From 129467d00e57f6cce34a8a4dc8b8b0e4a9b5e6e9 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 26 Apr 2016 00:48:25 -0300 Subject: [refactor] remove shared db locking from client Shared db locking was used to avoid the case in which two different devices try to store/modify remotelly stored secrets at the same time. We want to avoid remote locks because of the problems they create, and prefer to crash locally. For the record, we are currently using the user's password to encrypt the secrets stored in the server, and while we continue to do this we will have to re-encrypt the secrets and update the remote storage whenever the user changes her password. --- client/src/leap/soledad/client/secrets.py | 122 +++++++++++----------------- client/src/leap/soledad/client/shared_db.py | 30 ------- common/src/leap/soledad/common/errors.py | 61 -------------- 3 files changed, 49 insertions(+), 164 deletions(-) diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py index e2a5a1d7..a72aac0d 100644 --- a/client/src/leap/soledad/client/secrets.py +++ b/client/src/leap/soledad/client/secrets.py @@ -33,7 +33,6 @@ from hashlib import sha256 from leap.soledad.common import soledad_assert from leap.soledad.common import soledad_assert_type from leap.soledad.common import document -from leap.soledad.common import errors from leap.soledad.client import events from leap.soledad.client.crypto import encrypt_sym, decrypt_sym @@ -81,6 +80,7 @@ class BootstrapSequenceError(SecretsException): # Secrets handler # + class SoledadSecrets(object): """ @@ -162,17 +162,12 @@ class SoledadSecrets(object): :param shared_db: The shared database that stores user secrets. :type shared_db: leap.soledad.client.shared_db.SoledadSharedDatabase """ - # XXX removed since not in use - # We will pick the first secret available. - # param secret_id: The id of the storage secret to be used. - self._uuid = uuid self._userid = userid self._passphrase = passphrase self._secrets_path = secrets_path self._shared_db = shared_db self._secrets = {} - self._secret_id = None def bootstrap(self): @@ -197,47 +192,19 @@ class SoledadSecrets(object): # STAGE 1 - verify if secrets exist locally if not self._has_secret(): # try to load from local storage. - # STAGE 2 - there are no secrets in local storage, so try to fetch - # encrypted secrets from server. - logger.info( - 'Trying to fetch cryptographic secrets from shared recovery ' - 'database...') - - # --- start of atomic operation in shared db --- - - # obtain lock on shared db - token = timeout = None - try: - token, timeout = self._shared_db.lock() - except errors.AlreadyLockedError: - raise BootstrapSequenceError('Database is already locked.') - except errors.LockTimedOutError: - raise BootstrapSequenceError('Lock operation timed out.') + # STAGE 2 - there are no secrets in local storage and this is the + # first time we are running soledad with the specified + # secrets_path. Try to fetch encrypted secrets from + # server. + self._download_crypto_secrets() - self._get_or_gen_crypto_secrets() + if not self._has_secret(): - # release the lock on shared db - try: - self._shared_db.unlock(token) - self._shared_db.close() - except errors.NotLockedError: - # for some reason the lock expired. Despite that, secret - # loading or generation/storage must have been executed - # successfully, so we pass. - pass - except errors.InvalidTokenError: - # here, our lock has not only expired but also some other - # client application has obtained a new lock and is currently - # doing its thing in the shared database. Using the same - # reasoning as above, we assume everything went smooth and - # pass. - pass - except Exception as e: - logger.error("Unhandled exception when unlocking shared " - "database.") - logger.exception(e) - - # --- end of atomic operation in shared db --- + # STAGE 3 - there are no secrets in server also, so we want to + # generate the secrets and store them in the remote + # db. + self._gen_crypto_secrets() + self._upload_crypto_secrets() def _has_secret(self): """ @@ -295,13 +262,14 @@ class SoledadSecrets(object): self._store_secrets() self._put_secrets_in_shared_db() - def _get_or_gen_crypto_secrets(self): + def _download_crypto_secrets(self): """ - Retrieves or generates the crypto secrets. - - :raises BootstrapSequenceError: Raised when unable to store secrets in - shared database. + Downloads the crypto secrets. """ + logger.info( + 'Trying to fetch cryptographic secrets from shared recovery ' + 'database...') + if self._shared_db.syncable: doc = self._get_secrets_from_shared_db() else: @@ -314,31 +282,39 @@ class SoledadSecrets(object): _, active_secret = self._import_recovery_document(doc.content) self._maybe_set_active_secret(active_secret) self._store_secrets() # save new secrets in local file - else: - # STAGE 3 - there are no secrets in server also, so - # generate a secret and store it in remote db. - logger.info( - 'No cryptographic secrets found, creating new ' - ' secrets...') - self.set_secret_id(self._gen_secret()) - if self._shared_db.syncable: + def _gen_crypto_secrets(self): + """ + Generate the crypto secrets. + """ + logger.info('No cryptographic secrets found, creating new secrets...') + secret_id = self._gen_secret() + self.set_secret_id(secret_id) + + def _upload_crypto_secrets(self): + """ + Send crypto secrets to shared db. + + :raises BootstrapSequenceError: Raised when unable to store secrets in + shared database. + """ + if self._shared_db.syncable: + try: + self._put_secrets_in_shared_db() + except Exception as ex: + # storing generated secret in shared db failed for + # some reason, so we erase the generated secret and + # raise. try: - self._put_secrets_in_shared_db() - except Exception as ex: - # storing generated secret in shared db failed for - # some reason, so we erase the generated secret and - # raise. - try: - os.unlink(self._secrets_path) - except OSError as e: - if e.errno != errno.ENOENT: - # no such file or directory - logger.exception(e) - logger.exception(ex) - raise BootstrapSequenceError( - 'Could not store generated secret in the shared ' - 'database, bailing out...') + os.unlink(self._secrets_path) + except OSError as e: + if e.errno != errno.ENOENT: + # no such file or directory + logger.exception(e) + logger.exception(ex) + raise BootstrapSequenceError( + 'Could not store generated secret in the shared ' + 'database, bailing out...') # # Shared DB related methods diff --git a/client/src/leap/soledad/client/shared_db.py b/client/src/leap/soledad/client/shared_db.py index 6abf8ea3..a1d95fbe 100644 --- a/client/src/leap/soledad/client/shared_db.py +++ b/client/src/leap/soledad/client/shared_db.py @@ -151,33 +151,3 @@ class SoledadSharedDatabase(http_database.HTTPDatabase, TokenBasedAuth): http_database.HTTPDatabase.__init__(self, url, document_factory, creds) self._uuid = uuid - - def lock(self): - """ - Obtain a lock on document with id C{doc_id}. - - :return: A tuple containing the token to unlock and the timeout until - lock expiration. - :rtype: (str, float) - - :raise HTTPError: Raised if any HTTP error occurs. - """ - if self.syncable: - res, headers = self._request_json( - 'PUT', ['lock', self._uuid], body={}) - return res['token'], res['timeout'] - else: - return None, None - - def unlock(self, token): - """ - Release the lock on shared database. - - :param token: The token returned by a previous call to lock(). - :type token: str - - :raise HTTPError: - """ - if self.syncable: - _, _ = self._request_json( - 'DELETE', ['lock', self._uuid], params={'token': token}) diff --git a/common/src/leap/soledad/common/errors.py b/common/src/leap/soledad/common/errors.py index 0b6bb4e6..76a7240d 100644 --- a/common/src/leap/soledad/common/errors.py +++ b/common/src/leap/soledad/common/errors.py @@ -70,67 +70,6 @@ class InvalidAuthTokenError(errors.Unauthorized): status = 401 -# -# LockResource errors -# - -@register_exception -class InvalidTokenError(SoledadError): - - """ - Exception raised when trying to unlock shared database with invalid token. - """ - - wire_description = "unlock unauthorized" - status = 401 - - -@register_exception -class NotLockedError(SoledadError): - - """ - Exception raised when trying to unlock shared database when it is not - locked. - """ - - wire_description = "lock not found" - status = 404 - - -@register_exception -class AlreadyLockedError(SoledadError): - - """ - Exception raised when trying to lock shared database but it is already - locked. - """ - - wire_description = "lock is locked" - status = 403 - - -@register_exception -class LockTimedOutError(SoledadError): - - """ - Exception raised when timing out while trying to lock the shared database. - """ - - wire_description = "lock timed out" - status = 408 - - -@register_exception -class CouldNotObtainLockError(SoledadError): - - """ - Exception raised when timing out while trying to lock the shared database. - """ - - wire_description = "error obtaining lock" - status = 500 - - # # SoledadBackend errors # u1db error statuses also have to be updated -- cgit v1.2.3 From cf4453cb9325e7c6b3aec336fba1829c488a4442 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 26 Apr 2016 09:23:48 -0300 Subject: [refactor] remove shared db locking from server --- server/src/leap/soledad/server/__init__.py | 13 +- server/src/leap/soledad/server/auth.py | 5 - server/src/leap/soledad/server/lock_resource.py | 231 ------------------------ 3 files changed, 2 insertions(+), 247 deletions(-) delete mode 100644 server/src/leap/soledad/server/lock_resource.py diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 22894dac..43554eb4 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -49,9 +49,7 @@ synchronization is handled by the `leap.soledad.server.auth` module. Shared database --------------- -Each user may store password-encrypted recovery data in the shared database, -as well as obtain a lock on the shared database in order to prevent creation -of multiple secrets in parallel. +Each user may store password-encrypted recovery data in the shared database. Recovery documents are stored in the database without any information that may identify the user. In order to achieve this, the doc_id of recovery @@ -77,13 +75,8 @@ This has some implications: * Because of the above, it is recommended that recovery documents expire (not implemented yet) to prevent excess storage. -Lock documents, on the other hand, may be more thoroughly protected by the -server. Their doc_id's are calculated from the SHARED_DB_LOCK_DOC_ID_PREFIX -and the user's uid. - The authorization for creating, updating, deleting and retrieving recovery -and lock documents on the shared database is handled by -`leap.soledad.server.auth` module. +documents on the shared database is handled by `leap.soledad.server.auth` module. """ import configparser @@ -96,7 +89,6 @@ from ._version import get_versions from leap.soledad.server.auth import SoledadTokenAuthMiddleware from leap.soledad.server.gzip_middleware import GzipMiddleware -from leap.soledad.server.lock_resource import LockResource from leap.soledad.server.sync import ( SyncResource, MAX_REQUEST_SIZE, @@ -155,7 +147,6 @@ http_app.url_to_resource.register(http_app.DocsResource) http_app.url_to_resource.register(http_app.DocResource) # register Soledad's new or modified resources -http_app.url_to_resource.register(LockResource) http_app.url_to_resource.register(SyncResource) diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index ccbd6fbd..0ce1f497 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -101,7 +101,6 @@ class URLToAuthorization(object): /shared-db/docs | - /shared-db/doc/{any_id} | GET, PUT, DELETE /shared-db/sync-from/{source} | - - /shared-db/lock/{uuid} | PUT, DELETE /user-db | GET, PUT, DELETE /user-db/docs | - /user-db/doc/{id} | - @@ -118,10 +117,6 @@ class URLToAuthorization(object): '/%s/doc/{id:.*}' % SHARED_DB_NAME, [self.HTTP_METHOD_GET, self.HTTP_METHOD_PUT, self.HTTP_METHOD_DELETE]) - # auth info for shared-db lock resource - self._register( - '/%s/lock/%s' % (SHARED_DB_NAME, self._uuid), - [self.HTTP_METHOD_PUT, self.HTTP_METHOD_DELETE]) # auth info for user-db database resource self._register( '/%s' % self._user_db_name, diff --git a/server/src/leap/soledad/server/lock_resource.py b/server/src/leap/soledad/server/lock_resource.py deleted file mode 100644 index 0a602e26..00000000 --- a/server/src/leap/soledad/server/lock_resource.py +++ /dev/null @@ -1,231 +0,0 @@ -# -*- coding: utf-8 -*- -# lock_resource.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -LockResource: a lock based on a document in the shared database. -""" - - -import hashlib -import time -import os -import tempfile -import errno - - -from u1db.remote import http_app -from twisted.python.lockfile import FilesystemLock - - -from leap.soledad.common import ( - SHARED_DB_NAME, - SHARED_DB_LOCK_DOC_ID_PREFIX, -) -from leap.soledad.common.errors import ( - InvalidTokenError, - NotLockedError, - AlreadyLockedError, - LockTimedOutError, - CouldNotObtainLockError, -) - - -class LockResource(object): - """ - Handle requests for locking documents. - - This class uses Twisted's Filesystem lock to manage a lock in the shared - database. - """ - - url_pattern = '/%s/lock/{uuid}' % SHARED_DB_NAME - """ - """ - - TIMEOUT = 300 # XXX is 5 minutes reasonable? - """ - The timeout after which the lock expires. - """ - - # used for lock doc storage - TIMESTAMP_KEY = '_timestamp' - LOCK_TOKEN_KEY = '_token' - - FILESYSTEM_LOCK_TRIES = 5 - FILESYSTEM_LOCK_SLEEP_SECONDS = 1 - - def __init__(self, uuid, state, responder): - """ - Initialize the lock resource. Parameters to this constructor are - automatically passed by u1db. - - :param uuid: The user unique id. - :type uuid: str - :param state: The backend database state. - :type state: u1db.remote.ServerState - :param responder: The infrastructure to send responses to client. - :type responder: u1db.remote.HTTPResponder - """ - self._shared_db = state.open_database(SHARED_DB_NAME) - self._lock_doc_id = '%s%s' % (SHARED_DB_LOCK_DOC_ID_PREFIX, uuid) - self._lock = FilesystemLock( - os.path.join( - tempfile.gettempdir(), - hashlib.sha512(self._lock_doc_id).hexdigest())) - self._state = state - self._responder = responder - - @http_app.http_method(content=str) - def put(self, content=None): - """ - Handle a PUT request to the lock document. - - A lock is a document in the shared db with doc_id equal to - 'lock-' and the timestamp of its creation as content. This - method obtains a threaded-lock and creates a lock document if it does - not exist or if it has expired. - - It returns '201 Created' and a pair containing a token to unlock and - the lock timeout, or '403 AlreadyLockedError' and the remaining amount - of seconds the lock will still be valid. - - :param content: The content of the PUT request. It is only here - because PUT requests with empty content are considered - invalid requests by u1db. - :type content: str - """ - # obtain filesystem lock - if not self._try_obtain_filesystem_lock(): - self._responder.send_response_json( - LockTimedOutError.status, # error: request timeout - error=LockTimedOutError.wire_description) - return - - created_lock = False - now = time.time() - token = hashlib.sha256(os.urandom(10)).hexdigest() # for releasing - lock_doc = self._shared_db.get_doc(self._lock_doc_id) - remaining = self._remaining(lock_doc, now) - - # if there's no lock, create one - if lock_doc is None: - lock_doc = self._shared_db.create_doc( - { - self.TIMESTAMP_KEY: now, - self.LOCK_TOKEN_KEY: token, - }, - doc_id=self._lock_doc_id) - created_lock = True - else: - if remaining == 0: - # lock expired, create new one - lock_doc.content = { - self.TIMESTAMP_KEY: now, - self.LOCK_TOKEN_KEY: token, - } - self._shared_db.put_doc(lock_doc) - created_lock = True - - self._try_release_filesystem_lock() - - # send response to client - if created_lock is True: - self._responder.send_response_json( - 201, timeout=self.TIMEOUT, token=token) # success: created - else: - self._responder.send_response_json( - AlreadyLockedError.status, # error: forbidden - error=AlreadyLockedError.wire_description, remaining=remaining) - - @http_app.http_method(token=str) - def delete(self, token=None): - """ - Delete the lock if the C{token} is valid. - - Delete the lock document in case C{token} is equal to the token stored - in the lock document. - - :param token: The token returned when locking. - :type token: str - - :raise NotLockedError: Raised in case the lock is not locked. - :raise InvalidTokenError: Raised in case the token is invalid for - unlocking. - """ - lock_doc = self._shared_db.get_doc(self._lock_doc_id) - if lock_doc is None or self._remaining(lock_doc, time.time()) == 0: - self._responder.send_response_json( - NotLockedError.status, # error: not found - error=NotLockedError.wire_description) - elif token != lock_doc.content[self.LOCK_TOKEN_KEY]: - self._responder.send_response_json( - InvalidTokenError.status, # error: unauthorized - error=InvalidTokenError.wire_description) - else: - self._shared_db.delete_doc(lock_doc) - # respond success: should use 204 but u1db does not support it. - self._responder.send_response_json(200) - - def _remaining(self, lock_doc, now): - """ - Return the number of seconds the lock contained in C{lock_doc} is - still valid, when compared to C{now}. - - :param lock_doc: The document containing the lock. - :type lock_doc: u1db.Document - :param now: The time to which to compare the lock timestamp. - :type now: float - - :return: The amount of seconds the lock is still valid. - :rtype: float - """ - if lock_doc is not None: - lock_timestamp = lock_doc.content[self.TIMESTAMP_KEY] - remaining = lock_timestamp + self.TIMEOUT - now - return remaining if remaining > 0 else 0.0 - return 0.0 - - def _try_obtain_filesystem_lock(self): - """ - Try to obtain the file system lock. - - @return: Whether the lock was succesfully obtained. - @rtype: bool - """ - tries = self.FILESYSTEM_LOCK_TRIES - while tries > 0: - try: - return self._lock.lock() - except OSError as e: - tries -= 1 - if tries == 0: - raise CouldNotObtainLockError(e.message) - time.sleep(self.FILESYSTEM_LOCK_SLEEP_SECONDS) - return False - - def _try_release_filesystem_lock(self): - """ - Release the filesystem lock. - """ - try: - self._lock.unlock() - return True - except OSError as e: - if e.errno == errno.ENOENT: - return True - return False -- cgit v1.2.3 From daf515b0601f9ec2b0a8024b8a06a6814b45903d Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 26 Apr 2016 09:27:46 -0300 Subject: [refactor] remove shared db locking from tests --- .../src/leap/soledad/common/tests/test_server.py | 129 --------------------- .../src/leap/soledad/common/tests/test_soledad.py | 3 - common/src/leap/soledad/common/tests/util.py | 2 - 3 files changed, 134 deletions(-) diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py index 20fe8579..bf6c1515 100644 --- a/common/src/leap/soledad/common/tests/test_server.py +++ b/common/src/leap/soledad/common/tests/test_server.py @@ -43,7 +43,6 @@ from leap.soledad.common.tests.util import ( from leap.soledad.common import crypto from leap.soledad.client import Soledad -from leap.soledad.server import LockResource from leap.soledad.server import load_configuration from leap.soledad.server import CONFIG_DEFAULTS from leap.soledad.server.auth import URLToAuthorization @@ -494,134 +493,6 @@ class EncryptedSyncTestCase( return self._test_encrypted_sym_sync(doc_size=2, number_of_docs=100) -class LockResourceTestCase( - CouchDBTestCase, TestCaseWithServer): - - """ - Tests for use of PUT and DELETE on lock resource. - """ - - @staticmethod - def make_app_with_state(state): - return make_token_soledad_app(state) - - make_document_for_test = make_soledad_document_for_test - - sync_target = soledad_sync_target - - def setUp(self): - # the order of the following initializations is crucial because of - # dependencies. - # XXX explain better - CouchDBTestCase.setUp(self) - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - TestCaseWithServer.setUp(self) - # create the databases - db = CouchDatabase.open_database( - urljoin(self.couch_url, ('shared-%s' % (uuid4().hex))), - create=True, - ensure_ddocs=True) - self.addCleanup(db.delete_database) - self._state = CouchServerState(self.couch_url) - self._state.open_database = mock.Mock(return_value=db) - - def tearDown(self): - CouchDBTestCase.tearDown(self) - TestCaseWithServer.tearDown(self) - - def test__try_obtain_filesystem_lock(self): - responder = mock.Mock() - lock_uuid = uuid4().hex - lr = LockResource(lock_uuid, self._state, responder) - self.assertFalse(lr._lock.locked) - self.assertTrue(lr._try_obtain_filesystem_lock()) - self.assertTrue(lr._lock.locked) - lr._try_release_filesystem_lock() - - def test__try_release_filesystem_lock(self): - responder = mock.Mock() - lock_uuid = uuid4().hex - lr = LockResource(lock_uuid, self._state, responder) - lr._try_obtain_filesystem_lock() - self.assertTrue(lr._lock.locked) - lr._try_release_filesystem_lock() - self.assertFalse(lr._lock.locked) - - def test_put(self): - responder = mock.Mock() - lock_uuid = uuid4().hex - lr = LockResource(lock_uuid, self._state, responder) - # lock! - lr.put({}, None) - # assert lock document was correctly written - lock_doc = lr._shared_db.get_doc('lock-' + lock_uuid) - self.assertIsNotNone(lock_doc) - self.assertTrue(LockResource.TIMESTAMP_KEY in lock_doc.content) - self.assertTrue(LockResource.LOCK_TOKEN_KEY in lock_doc.content) - timestamp = lock_doc.content[LockResource.TIMESTAMP_KEY] - token = lock_doc.content[LockResource.LOCK_TOKEN_KEY] - self.assertTrue(timestamp < time.time()) - self.assertTrue(time.time() < timestamp + LockResource.TIMEOUT) - # assert response to user - responder.send_response_json.assert_called_with( - 201, token=token, - timeout=LockResource.TIMEOUT) - - def test_delete(self): - responder = mock.Mock() - lock_uuid = uuid4().hex - lr = LockResource(lock_uuid, self._state, responder) - # lock! - lr.put({}, None) - lock_doc = lr._shared_db.get_doc('lock-' + lock_uuid) - token = lock_doc.content[LockResource.LOCK_TOKEN_KEY] - # unlock! - lr.delete({'token': token}, None) - self.assertFalse(lr._lock.locked) - self.assertIsNone(lr._shared_db.get_doc('lock-' + lock_uuid)) - responder.send_response_json.assert_called_with(200) - - def test_put_while_locked_fails(self): - responder = mock.Mock() - lock_uuid = uuid4().hex - lr = LockResource(lock_uuid, self._state, responder) - # lock! - lr.put({}, None) - # try to lock again! - lr.put({}, None) - self.assertEqual( - len(responder.send_response_json.call_args), 2) - self.assertEqual( - responder.send_response_json.call_args[0], (403,)) - self.assertEqual( - len(responder.send_response_json.call_args[1]), 2) - self.assertTrue( - 'remaining' in responder.send_response_json.call_args[1]) - self.assertTrue( - responder.send_response_json.call_args[1]['remaining'] > 0) - - def test_unlock_unexisting_lock_fails(self): - responder = mock.Mock() - lock_uuid = uuid4().hex - lr = LockResource(lock_uuid, self._state, responder) - # unlock! - lr.delete({'token': 'anything'}, None) - responder.send_response_json.assert_called_with( - 404, error='lock not found') - - def test_unlock_with_wrong_token_fails(self): - responder = mock.Mock() - lock_uuid = uuid4().hex - lr = LockResource(lock_uuid, self._state, responder) - # lock! - lr.put({}, None) - # unlock! - lr.delete({'token': 'wrongtoken'}, None) - self.assertIsNotNone(lr._shared_db.get_doc('lock-' + lock_uuid)) - responder.send_response_json.assert_called_with( - 401, error='unlock unauthorized') - - class ConfigurationParsingTest(unittest.TestCase): def setUp(self): diff --git a/common/src/leap/soledad/common/tests/test_soledad.py b/common/src/leap/soledad/common/tests/test_soledad.py index 36c4003c..bf59ef8a 100644 --- a/common/src/leap/soledad/common/tests/test_soledad.py +++ b/common/src/leap/soledad/common/tests/test_soledad.py @@ -295,9 +295,6 @@ class SoledadSignalingTestCase(BaseSoledadTest): soledad.client.secrets.events.emit_async.assert_called_with( catalog.SOLEDAD_DONE_UPLOADING_KEYS, user_data ) - # assert db was locked and unlocked - sol.shared_db.lock.assert_called_with() - sol.shared_db.unlock.assert_called_with('atoken') sol.close() def test_stage2_bootstrap_signals(self): diff --git a/common/src/leap/soledad/common/tests/util.py b/common/src/leap/soledad/common/tests/util.py index d4510686..b3a08369 100644 --- a/common/src/leap/soledad/common/tests/util.py +++ b/common/src/leap/soledad/common/tests/util.py @@ -174,8 +174,6 @@ class MockedSharedDBTest(object): class defaultMockSharedDB(object): get_doc = Mock(return_value=get_doc_return_value) put_doc = Mock(side_effect=put_doc_side_effect) - lock = Mock(return_value=('atoken', 300)) - unlock = Mock(return_value=True) open = Mock(return_value=None) close = Mock(return_value=None) syncable = True -- cgit v1.2.3 From c634874aeeb4a9950e77ed28c8b8e643246e6bbd Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 26 Apr 2016 09:58:36 -0300 Subject: [refactor] cleanup bootstrap process --- client/src/leap/soledad/client/secrets.py | 89 +++++++++++----------- .../src/leap/soledad/common/tests/test_crypto.py | 9 --- 2 files changed, 45 insertions(+), 53 deletions(-) diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py index a72aac0d..16487572 100644 --- a/client/src/leap/soledad/client/secrets.py +++ b/client/src/leap/soledad/client/secrets.py @@ -190,21 +190,33 @@ class SoledadSecrets(object): storage on server sequence has failed for some reason. """ # STAGE 1 - verify if secrets exist locally - if not self._has_secret(): # try to load from local storage. - - # STAGE 2 - there are no secrets in local storage and this is the - # first time we are running soledad with the specified - # secrets_path. Try to fetch encrypted secrets from - # server. + try: + logger.info("Trying to load secrets from local storage...") + self._load_secrets_from_local_file() + logger.info("Found secrets in local storage.") + return + except NoStorageSecret: + logger.info("Could not find secrets in local storage.") + + # STAGE 2 - there are no secrets in local storage and this is the + # first time we are running soledad with the specified + # secrets_path. Try to fetch encrypted secrets from + # server. + try: + logger.info('Trying to fetch secrets from remote storage...') self._download_crypto_secrets() + logger.info('Found secrets in remote storage.') + return + except NoStorageSecret: + logger.info("Could not find secrets in remote storage.") - if not self._has_secret(): - - # STAGE 3 - there are no secrets in server also, so we want to - # generate the secrets and store them in the remote - # db. - self._gen_crypto_secrets() - self._upload_crypto_secrets() + # STAGE 3 - there are no secrets in server also, so we want to + # generate the secrets and store them in the remote + # db. + logger.info("Generating secrets...") + self._gen_crypto_secrets() + logger.info("Uploading secrets...") + self._upload_crypto_secrets() def _has_secret(self): """ @@ -213,21 +225,7 @@ class SoledadSecrets(object): :return: Whether there's a storage secret for symmetric encryption. :rtype: bool """ - logger.info("Checking if there's a secret in local storage...") - if (self._secret_id is None or self._secret_id not in self._secrets) \ - and os.path.isfile(self._secrets_path): - try: - self._load_secrets() # try to load from disk - except IOError as e: - logger.warning( - 'IOError while loading secrets from disk: %s' % str(e)) - - if self.storage_secret is not None: - logger.info("Found a secret in local storage.") - return True - - logger.info("Could not find a secret in local storage.") - return False + return self.storage_secret is not None def _maybe_set_active_secret(self, active_secret): """ @@ -239,10 +237,16 @@ class SoledadSecrets(object): active_secret = self._secrets.items()[0][0] self.set_secret_id(active_secret) - def _load_secrets(self): + def _load_secrets_from_local_file(self): """ Load storage secrets from local file. + :raise NoStorageSecret: Raised if there are no secrets available in + local storage. """ + # check if secrets file exists and we can read it + if not os.path.isfile(self._secrets_path): + raise NoStorageSecret + # read storage secrets from file content = None with open(self._secrets_path, 'r') as f: @@ -264,24 +268,21 @@ class SoledadSecrets(object): def _download_crypto_secrets(self): """ - Downloads the crypto secrets. - """ - logger.info( - 'Trying to fetch cryptographic secrets from shared recovery ' - 'database...') + Download crypto secrets. + :raise NoStorageSecret: Raised if there are no secrets available in + remote storage. + """ + doc = None if self._shared_db.syncable: doc = self._get_secrets_from_shared_db() - else: - doc = None - - if doc is not None: - logger.info( - 'Found cryptographic secrets in shared recovery ' - 'database.') - _, active_secret = self._import_recovery_document(doc.content) - self._maybe_set_active_secret(active_secret) - self._store_secrets() # save new secrets in local file + + if doc is None: + raise NoStorageSecret + + _, active_secret = self._import_recovery_document(doc.content) + self._maybe_set_active_secret(active_secret) + self._store_secrets() # save new secrets in local file def _gen_crypto_secrets(self): """ diff --git a/common/src/leap/soledad/common/tests/test_crypto.py b/common/src/leap/soledad/common/tests/test_crypto.py index ca10a1e1..4fc3161d 100644 --- a/common/src/leap/soledad/common/tests/test_crypto.py +++ b/common/src/leap/soledad/common/tests/test_crypto.py @@ -133,15 +133,6 @@ class SoledadSecretsTestCase(BaseSoledadTest): self.assertTrue( sol.secrets._has_secret(), "Should have a secret at this point") - # setting secret id to None should not interfere in the fact we have a - # secret. - sol.secrets.set_secret_id(None) - self.assertTrue( - sol.secrets._has_secret(), - "Should have a secret at this point") - # but not being able to decrypt correctly should - sol.secrets._secrets[sol.secrets.secret_id] = None - self.assertFalse(sol.secrets._has_secret()) sol.close() -- cgit v1.2.3 From af692e04c62b374c197d3ff45935ece5a7e100c1 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 26 Apr 2016 09:59:31 -0300 Subject: [refactor] remove old code for enlarging secrets --- client/src/leap/soledad/client/secrets.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py index 16487572..714b2dfe 100644 --- a/client/src/leap/soledad/client/secrets.py +++ b/client/src/leap/soledad/client/secrets.py @@ -253,18 +253,6 @@ class SoledadSecrets(object): content = json.loads(f.read()) _, active_secret = self._import_recovery_document(content) self._maybe_set_active_secret(active_secret) - # enlarge secret if needed - enlarged = False - if len(self._secrets[self._secret_id]) < self.GEN_SECRET_LENGTH: - gen_len = self.GEN_SECRET_LENGTH \ - - len(self._secrets[self._secret_id]) - new_piece = os.urandom(gen_len) - self._secrets[self._secret_id] += new_piece - enlarged = True - # store and save in shared db if needed - if enlarged: - self._store_secrets() - self._put_secrets_in_shared_db() def _download_crypto_secrets(self): """ -- cgit v1.2.3 From 96e27b6d258562d0e83696cefb1d11c60a31acf2 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 26 Apr 2016 22:43:27 -0300 Subject: [refactor] add changes file for shared db lock removal --- client/changes/next-changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/client/changes/next-changelog.rst b/client/changes/next-changelog.rst index bdc9f893..6e97386c 100644 --- a/client/changes/next-changelog.rst +++ b/client/changes/next-changelog.rst @@ -20,6 +20,7 @@ Bugfixes Misc ~~~~ +- Refactor bootstrap to remove shared db lock. - `#1236 `_: Description of the new feature corresponding with issue #1236. - Some change without issue number. -- cgit v1.2.3 From ff5bd3f8e1cedd01119cd02b395c2bdbfa377c71 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Mon, 16 May 2016 15:49:18 -0400 Subject: [style] pep8 --- client/src/leap/soledad/client/crypto.py | 3 ++- common/src/leap/soledad/common/__init__.py | 4 ---- server/src/leap/soledad/server/__init__.py | 3 ++- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index 363d71b9..b75d4301 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -26,7 +26,8 @@ import logging from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from cryptography.hazmat.backends.multibackend import MultiBackend -from cryptography.hazmat.backends.openssl.backend import Backend as OpenSSLBackend +from cryptography.hazmat.backends.openssl.backend \ + import Backend as OpenSSLBackend from leap.soledad.common import soledad_assert from leap.soledad.common import soledad_assert_type diff --git a/common/src/leap/soledad/common/__init__.py b/common/src/leap/soledad/common/__init__.py index d7f6929c..1ba6ab89 100644 --- a/common/src/leap/soledad/common/__init__.py +++ b/common/src/leap/soledad/common/__init__.py @@ -47,7 +47,3 @@ __all__ = [ "soledad_assert_type", "__version__", ] - -from ._version import get_versions -__version__ = get_versions()['version'] -del get_versions diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 43554eb4..72d61224 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -76,7 +76,8 @@ This has some implications: (not implemented yet) to prevent excess storage. The authorization for creating, updating, deleting and retrieving recovery -documents on the shared database is handled by `leap.soledad.server.auth` module. +documents on the shared database is handled by `leap.soledad.server.auth` +module. """ import configparser -- cgit v1.2.3 From 695c827db9b60d9a9b368232aac8912b1270bebe Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 18 May 2016 12:01:37 -0400 Subject: [pkg] update to new versioneer json format --- common/setup.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/common/setup.py b/common/setup.py index 8d9c4d6e..c1f4d5ac 100644 --- a/common/setup.py +++ b/common/setup.py @@ -75,14 +75,20 @@ class freeze_debianver(Command): # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. -version_version = '{version}' -full_revisionid = '{full_revisionid}' -""" - templatefun = r""" - -def get_versions(default={}, verbose=False): - return {'version': version_version, - 'full-revisionid': full_revisionid} +import json +import sys + +version_json = ''' +{ + "dirty": false, + "error": null, + "full-revisionid": "FULL_REVISIONID", + "version": "VERSION_STRING" +} +''' # END VERSION_JSON + +def get_versions(): + return json.loads(version_json) """ def initialize_options(self): @@ -97,14 +103,13 @@ def get_versions(default={}, verbose=False): if proceed != "y": print("He. You scared. Aborting.") return - subst_template = self.template.format( - version=VERSION_SHORT, - full_revisionid=VERSION_REVISION) + self.templatefun + subst_template = self.template.replace( + 'VERSION_STRING', VERSION_SHORT).replace( + 'FULL_REVISIONID', VERSION_REVISION) versioneer_cfg = versioneer.get_config_from_root('.') with open(versioneer_cfg.versionfile_source, 'w') as f: f.write(subst_template) - cmdclass = versioneer.get_cmdclass() # -- cgit v1.2.3 From 21dbbc534be2c4668011cc9e631a7e4ba24061fa Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 18 May 2016 12:04:59 -0400 Subject: [pkg] update to new versioneer json format --- client/setup.py | 28 +++++++++++++++++----------- server/setup.py | 28 +++++++++++++++++----------- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/client/setup.py b/client/setup.py index 4480e247..90986dde 100644 --- a/client/setup.py +++ b/client/setup.py @@ -68,14 +68,20 @@ class freeze_debianver(Command): # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. -version_version = '{version}' -full_revisionid = '{full_revisionid}' -""" - templatefun = r""" - -def get_versions(default={}, verbose=False): - return {'version': version_version, - 'full-revisionid': full_revisionid} +import json +import sys + +version_json = ''' +{ + "dirty": false, + "error": null, + "full-revisionid": "FULL_REVISIONID", + "version": "VERSION_STRING" +} +''' # END VERSION_JSON + +def get_versions(): + return json.loads(version_json) """ def initialize_options(self): @@ -90,9 +96,9 @@ def get_versions(default={}, verbose=False): if proceed != "y": print("He. You scared. Aborting.") return - subst_template = self.template.format( - version=VERSION_SHORT, - full_revisionid=VERSION_REVISION) + self.templatefun + subst_template = self.template.replace( + 'VERSION_STRING', VERSION_SHORT).replace( + 'FULL_REVISIONID', VERSION_REVISION) versioneer_cfg = versioneer.get_config_from_root('.') with open(versioneer_cfg.versionfile_source, 'w') as f: f.write(subst_template) diff --git a/server/setup.py b/server/setup.py index 8a7fbe45..b3b26010 100644 --- a/server/setup.py +++ b/server/setup.py @@ -77,14 +77,20 @@ class freeze_debianver(Command): # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. -version_version = '{version}' -full_revisionid = '{full_revisionid}' -""" - templatefun = r""" - -def get_versions(default={}, verbose=False): - return {'version': version_version, - 'full-revisionid': full_revisionid} +import json +import sys + +version_json = ''' +{ + "dirty": false, + "error": null, + "full-revisionid": "FULL_REVISIONID", + "version": "VERSION_STRING" +} +''' # END VERSION_JSON + +def get_versions(): + return json.loads(version_json) """ def initialize_options(self): @@ -99,9 +105,9 @@ def get_versions(default={}, verbose=False): if proceed != "y": print("He. You scared. Aborting.") return - subst_template = self.template.format( - version=VERSION_SHORT, - full_revisionid=VERSION_REVISION) + self.templatefun + subst_template = self.template.replace( + 'VERSION_STRING', VERSION_SHORT).replace( + 'FULL_REVISIONID', VERSION_REVISION) versioneer_cfg = versioneer.get_config_from_root('.') with open(versioneer_cfg.versionfile_source, 'w') as f: f.write(subst_template) -- cgit v1.2.3 From 66e3572959774449d4efca5b72efe41af54075e7 Mon Sep 17 00:00:00 2001 From: Caio Carrara Date: Thu, 14 Apr 2016 22:12:53 -0300 Subject: [refactor] remove user_id argument from Soledad init The constructor method of Soledad was receiving two arguments for user id. One of them was optional with None as default. It could cause an inconsistent state with uuid set but userid unset. This change remove the optional user_id argument from initialization method and return the uuid if anyone call Soledad.userid method. --- client/src/leap/soledad/client/api.py | 8 +++----- common/src/leap/soledad/common/tests/test_soledad.py | 3 +-- common/src/leap/soledad/common/tests/util.py | 6 ++---- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index e657c939..2477350e 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -126,8 +126,7 @@ class Soledad(object): def __init__(self, uuid, passphrase, secrets_path, local_db_path, server_url, cert_file, shared_db=None, - auth_token=None, defer_encryption=False, syncable=True, - userid=None): + auth_token=None, defer_encryption=False, syncable=True): """ Initialize configuration, cryptographic keys and dbs. @@ -181,7 +180,6 @@ class Soledad(object): """ # store config params self._uuid = uuid - self._userid = userid self._passphrase = passphrase self._local_db_path = local_db_path self._server_url = server_url @@ -255,7 +253,7 @@ class Soledad(object): """ self._secrets = SoledadSecrets( self.uuid, self._passphrase, self._secrets_path, - self.shared_db, userid=self._userid) + self.shared_db, userid=self.userid) self._secrets.bootstrap() def _init_u1db_sqlcipher_backend(self): @@ -655,7 +653,7 @@ class Soledad(object): @property def userid(self): - return self._userid + return self.uuid # # ISyncableStorage diff --git a/common/src/leap/soledad/common/tests/test_soledad.py b/common/src/leap/soledad/common/tests/test_soledad.py index bf59ef8a..b48915eb 100644 --- a/common/src/leap/soledad/common/tests/test_soledad.py +++ b/common/src/leap/soledad/common/tests/test_soledad.py @@ -249,8 +249,7 @@ class SoledadSignalingTestCase(BaseSoledadTest): # get a fresh instance so it emits all bootstrap signals sol = self._soledad_instance( secrets_path='alternative_stage3.json', - local_db_path='alternative_stage3.u1db', - userid=ADDRESS) + local_db_path='alternative_stage3.u1db') # reverse call order so we can verify in the order the signals were # expected soledad.client.secrets.events.emit_async.mock_calls.reverse() diff --git a/common/src/leap/soledad/common/tests/util.py b/common/src/leap/soledad/common/tests/util.py index b3a08369..c681fa93 100644 --- a/common/src/leap/soledad/common/tests/util.py +++ b/common/src/leap/soledad/common/tests/util.py @@ -284,8 +284,7 @@ class BaseSoledadTest(BaseLeapTest, MockedSharedDBTest): server_url='https://127.0.0.1/', cert_file=None, shared_db_class=None, - auth_token='auth-token', - userid=ADDRESS): + auth_token='auth-token'): def _put_doc_side_effect(doc): self._doc_put = doc @@ -307,8 +306,7 @@ class BaseSoledadTest(BaseLeapTest, MockedSharedDBTest): cert_file=cert_file, defer_encryption=self.defer_sync_encryption, shared_db=MockSharedDB(), - auth_token=auth_token, - userid=userid) + auth_token=auth_token) self.addCleanup(soledad.close) return soledad -- cgit v1.2.3 From 1fd07f013736599159c364b774ca5b3c6c6747c0 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 7 Apr 2016 16:00:04 -0400 Subject: [feature] debug-mode server with dummy authentication to ease debugging of local servers w/o neededing the Token machinery in place. this needs still some extra changes to be fully functional: - adapt the create-userdb script to work with no auth info. --- server/src/leap/soledad/server/__init__.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 72d61224..195714c1 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -304,16 +304,34 @@ def load_configuration(file_path): # Run as Twisted WSGI Resource # ---------------------------------------------------------------------------- -def application(environ, start_response): + +def _load_config(): conf = load_configuration('/etc/soledad/soledad-server.conf') - conf = conf['soledad-server'] + return conf['soledad-server'] + + +def _get_couch_state(): + conf = _load_config() state = CouchServerState(conf['couch_url'], create_cmd=conf['create_cmd']) - SoledadBackend.BATCH_SUPPORT = conf['batching'] - # WSGI application that may be used by `twistd -web` + SoledadBackend.BATCH_SUPPORT = conf.get('batching', False) + return state + + +def application(environ, start_response): + """return WSGI application that may be used by `twistd -web`""" + state = _get_couch_state() application = GzipMiddleware( SoledadTokenAuthMiddleware(SoledadApp(state))) + return application(environ, start_response) + +def debug_local_application_do_not_use(environ, start_response): + """in where we bypass token auth middleware for ease of mind while + debugging in your local environment""" + state = _get_couch_state() + application = SoledadApp(state) return application(environ, start_response) + __version__ = get_versions()['version'] del get_versions -- cgit v1.2.3 From 4c1269f37ad67337d9ad65bcb24ccc5e8eeb1950 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 8 Apr 2016 02:14:20 -0400 Subject: [refactor] adapt create-user-db script to bypass auth for local tests --- server/pkg/create-user-db | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/server/pkg/create-user-db b/server/pkg/create-user-db index 54856643..5e48d4de 100755 --- a/server/pkg/create-user-db +++ b/server/pkg/create-user-db @@ -25,6 +25,9 @@ from leap.soledad.common.couch import list_users_dbs from leap.soledad.server import load_configuration +BYPASS_AUTH = os.environ.get('SOLEDAD_BYPASS_AUTH', False) + + description = """ Creates a user database. This is meant to be used by Soledad Server. @@ -40,16 +43,23 @@ NETRC_PATH = CONF['soledad-server']['admin_netrc'] def url_for_db(dbname): - if not os.path.exists(NETRC_PATH): - print ('netrc not found in %s' % NETRC_PATH) - sys.exit(1) - parsed_netrc = netrc.netrc(NETRC_PATH) - host, (login, _, password) = parsed_netrc.hosts.items()[0] - url = ('http://%(login)s:%(password)s@%(host)s:5984/%(dbname)s' % { - 'login': login, - 'password': password, - 'host': host, - 'dbname': dbname}) + if BYPASS_AUTH: + login = '' + password = '' + host = 'localhost' + url = 'http://localhost:5984/%(dbname)s' % { + 'dbname': dbname} + else: + if not os.path.exists(NETRC_PATH): + print ('netrc not found in %s' % NETRC_PATH) + sys.exit(1) + parsed_netrc = netrc.netrc(NETRC_PATH) + host, (login, _, password) = parsed_netrc.hosts.items()[0] + url = ('http://%(login)s:%(password)s@%(host)s:5984/%(dbname)s' % { + 'login': login, + 'password': password, + 'host': host, + 'dbname': dbname}) return url -- cgit v1.2.3 From 9eac24ba91ccbbd335e2cf6d8f59c518659348e6 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Fri, 8 Apr 2016 01:44:49 -0400 Subject: [refactor] adapt profiling script to local debug server --- scripts/profiling/sync/profile-sync.py | 49 +++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/scripts/profiling/sync/profile-sync.py b/scripts/profiling/sync/profile-sync.py index 8c18bde8..34e66f03 100755 --- a/scripts/profiling/sync/profile-sync.py +++ b/scripts/profiling/sync/profile-sync.py @@ -1,4 +1,10 @@ #!/usr/bin/env python +""" +Example of usage: + time ./profile-sync.py --no-stats --send-num 5 --payload-file sample \ + --repeat-payload -p password -b /tmp/foobarsync \ + test_soledad_sync_001@cdev.bitmask.net +""" import argparse import commands @@ -13,7 +19,9 @@ from twisted.internet import reactor from util import StatsLogger, ValidateUserHandle from client_side_db import _get_soledad_instance, _get_soledad_info + from leap.common.events import flags +from leap.soledad.client.api import Soledad flags.set_events_enabled(False) @@ -70,6 +78,23 @@ def create_docs(soledad, args): payload = fmap.read(docsize * 1024) s.create_doc({payload: payload}) + +def _get_soledad_instance_from_uuid(uuid, passphrase, basedir, server_url, + cert_file, token): + secrets_path = os.path.join(basedir, '%s.secret' % uuid) + local_db_path = os.path.join(basedir, '%s.db' % uuid) + return Soledad( + uuid, + unicode(passphrase), + secrets_path=secrets_path, + local_db_path=local_db_path, + server_url=server_url, + cert_file=cert_file, + auth_token=token, + defer_encryption=True, + syncable=True) + + # main program if __name__ == '__main__': @@ -78,6 +103,9 @@ if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( 'user@provider', action=ValidateUserHandle, help='the user handle') + parser.add_argument( + '-u', dest='uuid', required=False, default=None, + help='uuid for local tests') parser.add_argument( '-b', dest='basedir', required=False, default=None, help='soledad base directory') @@ -102,6 +130,7 @@ if __name__ == '__main__': parser.add_argument( '--payload-file', dest="payload_f", default=None, help='path to a sample file to use for the payloads') + parser.add_argument( '--no-stats', dest='do_stats', action='store_false', help='skip system stats') @@ -132,12 +161,20 @@ if __name__ == '__main__': basedir = tempfile.mkdtemp() logger.info('Using %s as base directory.' % basedir) - uuid, server_url, cert_file, token = \ - _get_soledad_info( - args.username, args.provider, passphrase, basedir) - # get the soledad instance - s = _get_soledad_instance( - uuid, passphrase, basedir, server_url, cert_file, token) + if args.uuid: + # We got an uuid. This is a local test, and we bypass + # authentication and encryption. + s = _get_soledad_instance_from_uuid( + args.uuid, passphrase, basedir, 'http://localhost:2323', '', '') + + else: + # Remote server. First, get remote info... + uuid, server_url, cert_file, token = \ + _get_soledad_info( + args.username, args.provider, passphrase, basedir) + # ...and then get the soledad instance + s = _get_soledad_instance( + uuid, passphrase, basedir, server_url, cert_file, token) if args.do_send: create_docs(s, args) -- cgit v1.2.3 From 06f3c80e848b14d3fff1a6edd2cd58f998b976db Mon Sep 17 00:00:00 2001 From: drebs Date: Sun, 1 May 2016 13:34:33 -0300 Subject: [bug] remove doc content conversion to unicode Theoretically (until now), Soledad inherits from U1DB the behaviour of only accepting valid JSON for documents contents. JSON documents only allow for unicode strings. Despite that, until now we had implemented lossy convertion to unicode to avoid encoding errors when dumping/loading JSON content. This allowed for API users to pass non-unicode to Soledad, but caused the application to take more time because of conversion. There were 2 problem with this: (1) conversion may take a long time and a lot of memory when convertin large payloads; and (2) conversion was being made before deferring to the adbapi, and this was blocking the reactor. This commit completelly removes the conversion to unicode, thus leaving the responsibility of unicode conversion to users of the Soledad API. --- client/changes/next-changelog.rst | 2 ++ client/src/leap/soledad/client/api.py | 46 +---------------------------------- 2 files changed, 3 insertions(+), 45 deletions(-) diff --git a/client/changes/next-changelog.rst b/client/changes/next-changelog.rst index 6e97386c..050d84be 100644 --- a/client/changes/next-changelog.rst +++ b/client/changes/next-changelog.rst @@ -16,6 +16,8 @@ Features Bugfixes ~~~~~~~~ - `#1235 `_: Description for the fixed stuff corresponding with issue #1235. +- Remove document content conversion to unicode. Users of API are responsible + for only passing valid JSON to Soledad for storage. - Bugfix without related issue number. Misc diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 2477350e..d83291e7 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -35,10 +35,6 @@ import ssl import uuid import urlparse -try: - import cchardet as chardet -except ImportError: - import chardet from itertools import chain from StringIO import StringIO @@ -357,7 +353,6 @@ class Soledad(object): also be updated. :rtype: twisted.internet.defer.Deferred """ - doc.content = _convert_to_unicode(doc.content) return self._defer("put_doc", doc) def delete_doc(self, doc): @@ -452,8 +447,7 @@ class Soledad(object): # create_doc (and probably to put_doc too). There are cases (mail # payloads for example) in which we already have the encoding in the # headers, so we don't need to guess it. - return self._defer( - "create_doc", _convert_to_unicode(content), doc_id=doc_id) + return self._defer("create_doc", content, doc_id=doc_id) def create_doc_from_json(self, json, doc_id=None): """ @@ -974,44 +968,6 @@ class Soledad(object): return self.create_doc(doc) -def _convert_to_unicode(content): - """ - Convert content to unicode (or all the strings in content). - - NOTE: Even though this method supports any type, it will - currently ignore contents of lists, tuple or any other - iterable than dict. We don't need support for these at the - moment - - :param content: content to convert - :type content: object - - :rtype: object - """ - # Chardet doesn't guess very well with some smallish payloads. - # This parameter might need some empirical tweaking. - CUTOFF_CONFIDENCE = 0.90 - - if isinstance(content, unicode): - return content - elif isinstance(content, str): - encoding = "utf-8" - result = chardet.detect(content) - if result["confidence"] > CUTOFF_CONFIDENCE: - encoding = result["encoding"] - try: - content = content.decode(encoding) - except UnicodeError as e: - logger.error("Unicode error: {0!r}. Using 'replace'".format(e)) - content = content.decode(encoding, 'replace') - return content - else: - if isinstance(content, dict): - for key in content.keys(): - content[key] = _convert_to_unicode(content[key]) - return content - - def create_path_if_not_exists(path): try: if not os.path.isdir(path): -- cgit v1.2.3 From 951593776e6dabdeef69b4138e4cc3d789e6295f Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Sun, 1 May 2016 13:58:28 -0400 Subject: [feature] use deferred semaphore --- client/changes/next-changelog.rst | 1 + client/src/leap/soledad/client/adbapi.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/client/changes/next-changelog.rst b/client/changes/next-changelog.rst index 050d84be..a696fe10 100644 --- a/client/changes/next-changelog.rst +++ b/client/changes/next-changelog.rst @@ -10,6 +10,7 @@ I've added a new category `Misc` so we can track doc/style/packaging stuff. Features ~~~~~~~~ +- Use DeferredLock instead of its locking cousin. - `#1234 `_: Description of the new feature corresponding with issue #1234. - New feature without related issue number. diff --git a/client/src/leap/soledad/client/adbapi.py b/client/src/leap/soledad/client/adbapi.py index 77822247..f43e8110 100644 --- a/client/src/leap/soledad/client/adbapi.py +++ b/client/src/leap/soledad/client/adbapi.py @@ -24,9 +24,9 @@ import sys import logging from functools import partial -from threading import BoundedSemaphore from twisted.enterprise import adbapi +from twisted.internet.defer import DeferredSemaphore from twisted.python import log from zope.proxy import ProxyBase, setProxiedObject from pysqlcipher.dbapi2 import OperationalError @@ -204,7 +204,7 @@ class U1DBConnectionPool(adbapi.ConnectionPool): :rtype: twisted.internet.defer.Deferred """ meth = "u1db_%s" % meth - semaphore = BoundedSemaphore(SQLCIPHER_MAX_RETRIES - 1) + semaphore = DeferredSemaphore(SQLCIPHER_MAX_RETRIES ) def _run_interaction(): return self.runInteraction( @@ -213,7 +213,7 @@ class U1DBConnectionPool(adbapi.ConnectionPool): def _errback(failure): failure.trap(OperationalError) if failure.getErrorMessage() == "database is locked": - should_retry = semaphore.acquire(False) + should_retry = semaphore.acquire() if should_retry: logger.warning( "Database operation timed out while waiting for " -- cgit v1.2.3 From 50f90874815a85eb82e0fba2d953b680ee9eeb53 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Wed, 3 Feb 2016 11:01:23 -0300 Subject: [refactor] bye multiprocessing pool This commit removes the multiprocessing pool and gives a step closer to make encdecpool simpler. Download speed is now at a constant rate, CPU usage lower and reactor responding fast when running with a HTTP server like Pixelated. --- client/src/leap/soledad/client/encdecpool.py | 117 ++------------------- .../src/leap/soledad/client/http_target/fetch.py | 9 +- 2 files changed, 14 insertions(+), 112 deletions(-) diff --git a/client/src/leap/soledad/client/encdecpool.py b/client/src/leap/soledad/client/encdecpool.py index 34667a1e..576b8b2c 100644 --- a/client/src/leap/soledad/client/encdecpool.py +++ b/client/src/leap/soledad/client/encdecpool.py @@ -22,8 +22,6 @@ during synchronization. """ -import multiprocessing -import Queue import json import logging @@ -51,9 +49,6 @@ class SyncEncryptDecryptPool(object): Base class for encrypter/decrypter pools. """ - # TODO implement throttling to reduce cpu usage?? - WORKERS = multiprocessing.cpu_count() - def __init__(self, crypto, sync_db): """ Initialize the pool of encryption-workers. @@ -66,21 +61,18 @@ class SyncEncryptDecryptPool(object): """ self._crypto = crypto self._sync_db = sync_db - self._pool = None self._delayed_call = None self._started = False def start(self): if self.running: return - self._create_pool() self._started = True def stop(self): if not self.running: return self._started = False - self._destroy_pool() # maybe cancel the next delayed call if self._delayed_call \ and not self._delayed_call.called: @@ -90,27 +82,6 @@ class SyncEncryptDecryptPool(object): def running(self): return self._started - def _create_pool(self): - self._pool = multiprocessing.Pool(self.WORKERS) - - def _destroy_pool(self): - """ - Cleanly close the pool of workers. - """ - logger.debug("Closing %s" % (self.__class__.__name__,)) - self._pool.close() - try: - self._pool.join() - except Exception: - pass - - def terminate(self): - """ - Terminate the pool of workers. - """ - logger.debug("Terminating %s" % (self.__class__.__name__,)) - self._pool.terminate() - def _runOperation(self, query, *args): """ Run an operation on the sync db. @@ -180,7 +151,6 @@ class SyncEncrypterPool(SyncEncryptDecryptPool): Initialize the sync encrypter pool. """ SyncEncryptDecryptPool.__init__(self, *args, **kwargs) - self._encr_queue = defer.DeferredQueue() # TODO delete already synced files from database def start(self): @@ -189,19 +159,11 @@ class SyncEncrypterPool(SyncEncryptDecryptPool): """ SyncEncryptDecryptPool.start(self) logger.debug("Starting the encryption loop...") - reactor.callWhenRunning(self._maybe_encrypt_and_recurse) def stop(self): """ Stop the encrypter pool. """ - # close the sync queue - if self._encr_queue: - q = self._encr_queue - for d in q.pending: - d.cancel() - del q - self._encr_queue = None SyncEncryptDecryptPool.stop(self) @@ -212,29 +174,7 @@ class SyncEncrypterPool(SyncEncryptDecryptPool): :param doc: The document to be encrypted. :type doc: SoledadDocument """ - try: - self._encr_queue.put(doc) - except Queue.Full: - # do not asynchronously encrypt this file if the queue is full - pass - - @defer.inlineCallbacks - def _maybe_encrypt_and_recurse(self): - """ - Process one document from the encryption queue. - - Asynchronously encrypt a document that will then be stored in the sync - db. Processed documents will be read by the SoledadSyncTarget during - the sync_exchange. - """ - try: - while self.running: - doc = yield self._encr_queue.get() - self._encrypt_doc(doc) - except defer.QueueUnderflow: - self._delayed_call = reactor.callLater( - self.ENCRYPT_LOOP_PERIOD, - self._maybe_encrypt_and_recurse) + self._encrypt_doc(doc) def _encrypt_doc(self, doc): """ @@ -253,9 +193,9 @@ class SyncEncrypterPool(SyncEncryptDecryptPool): secret = self._crypto.secret args = doc.doc_id, doc.rev, docstr, key, secret # encrypt asynchronously - self._pool.apply_async( - encrypt_doc_task, args, - callback=self._encrypt_doc_cb) + d = threads.deferToThread( + encrypt_doc_task, *args) + d.addCallback(self._encrypt_doc_cb) def _encrypt_doc_cb(self, result): """ @@ -354,6 +294,7 @@ def decrypt_doc_task(doc_id, doc_rev, content, gen, trans_id, key, secret, :return: A tuple containing the doc id, revision and encrypted content. :rtype: tuple(str, str, str) """ + content = json.loads(content) if type(content) == str else content decrypted_content = decrypt_doc_dict(content, doc_id, doc_rev, key, secret) return doc_id, doc_rev, decrypted_content, gen, trans_id, idx @@ -414,7 +355,6 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): self._docs_to_process = None self._processed_docs = 0 self._last_inserted_idx = 0 - self._decrypting_docs = [] # a list that holds the asynchronous decryption results so they can be # collected when they are ready @@ -511,11 +451,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): has finished. :rtype: twisted.internet.defer.Deferred """ - docstr = json.dumps(content) - query = "INSERT OR REPLACE INTO '%s' VALUES (?, ?, ?, ?, ?, ?, ?)" \ - % self.TABLE_NAME - return self._runOperation( - query, (doc_id, doc_rev, docstr, gen, trans_id, 1, idx)) + return self._async_decrypt_doc(doc_id, doc_rev, content, gen, trans_id, idx) def insert_received_doc( self, doc_id, doc_rev, content, gen, trans_id, idx): @@ -585,14 +521,12 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): """ soledad_assert(self._crypto is not None, "need a crypto object") - content = json.loads(content) key = self._crypto.doc_passphrase(doc_id) secret = self._crypto.secret args = doc_id, rev, content, gen, trans_id, key, secret, idx # decrypt asynchronously - self._async_results.append( - self._pool.apply_async( - decrypt_doc_task, args)) + d = threads.deferToThread(decrypt_doc_task, *args) + d.addCallback(self._decrypt_doc_cb) def _decrypt_doc_cb(self, result): """ @@ -610,7 +544,6 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): doc_id, rev, content, gen, trans_id, idx = result logger.debug("Sync decrypter pool: decrypted doc %s: %s %s %s" % (doc_id, rev, gen, trans_id)) - self._decrypting_docs.remove((doc_id, rev)) return self.insert_received_doc( doc_id, rev, content, gen, trans_id, idx) @@ -659,23 +592,6 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): last_idx += 1 defer.returnValue(insertable) - @defer.inlineCallbacks - def _async_decrypt_received_docs(self): - """ - Get all the encrypted documents from the sync database and dispatch a - decrypt worker to decrypt each one of them. - - :return: A deferred that will fire after all documents have been - decrypted and inserted back in the sync db. - :rtype: twisted.internet.defer.Deferred - """ - docs = yield self._get_docs(encrypted=True) - for doc_id, rev, content, gen, trans_id, _, idx in docs: - if (doc_id, rev) not in self._decrypting_docs: - self._decrypting_docs.append((doc_id, rev)) - self._async_decrypt_doc( - doc_id, rev, content, gen, trans_id, idx) - @defer.inlineCallbacks def _process_decrypted_docs(self): """ @@ -762,21 +678,6 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): query = "DELETE FROM %s WHERE 1" % (self.TABLE_NAME,) return self._runOperation(query) - @defer.inlineCallbacks - def _collect_async_decryption_results(self): - """ - Collect the results of the asynchronous doc decryptions and re-raise - any exception raised by a multiprocessing async decryption call. - - :raise Exception: Raised if an async call has raised an exception. - """ - async_results = self._async_results[:] - for res in async_results: - if res.ready(): - # XXX: might raise an exception! - yield self._decrypt_doc_cb(res.get()) - self._async_results.remove(res) - @defer.inlineCallbacks def _decrypt_and_recurse(self): """ @@ -796,8 +697,6 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): pending = self._docs_to_process if processed < pending: - yield self._async_decrypt_received_docs() - yield self._collect_async_decryption_results() docs = yield self._process_decrypted_docs() yield self._delete_processed_docs(docs) # recurse diff --git a/client/src/leap/soledad/client/http_target/fetch.py b/client/src/leap/soledad/client/http_target/fetch.py index 9f7a4193..fda90909 100644 --- a/client/src/leap/soledad/client/http_target/fetch.py +++ b/client/src/leap/soledad/client/http_target/fetch.py @@ -19,6 +19,7 @@ import json from u1db import errors from u1db.remote import utils from twisted.internet import defer +from twisted.internet import threads from leap.soledad.common.document import SoledadDocument from leap.soledad.client.events import SOLEDAD_SYNC_RECEIVE_STATUS from leap.soledad.client.events import emit_async @@ -75,7 +76,7 @@ class HTTPDocFetcher(object): last_known_generation, last_known_trans_id, sync_id, 0) self._received_docs = 0 - number_of_changes, ngen, ntrans = self._insert_received_doc(doc, 1, 1) + number_of_changes, ngen, ntrans = yield self._insert_received_doc(doc, 1, 1) if ngen: new_generation = ngen @@ -137,6 +138,7 @@ class HTTPDocFetcher(object): body=str(body), content_type='application/x-soledad-sync-get') + @defer.inlineCallbacks def _insert_received_doc(self, response, idx, total): """ Insert a received document into the local replica. @@ -150,7 +152,8 @@ class HTTPDocFetcher(object): """ new_generation, new_transaction_id, number_of_changes, doc_id, \ rev, content, gen, trans_id = \ - self._parse_received_doc_response(response) + (yield threads.deferToThread(self._parse_received_doc_response, + response)) if doc_id is not None: # decrypt incoming document and insert into local database # ------------------------------------------------------------- @@ -185,7 +188,7 @@ class HTTPDocFetcher(object): self._received_docs += 1 user_data = {'uuid': self.uuid, 'userid': self.userid} _emit_receive_status(user_data, self._received_docs, total) - return number_of_changes, new_generation, new_transaction_id + defer.returnValue((number_of_changes, new_generation, new_transaction_id)) def _parse_received_doc_response(self, response): """ -- cgit v1.2.3 From 498e9e1353700b61950ef87c007e6c0a84fbe201 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Thu, 4 Feb 2016 13:33:49 -0300 Subject: [refactor] encdecpool queries and testing This commit adds tests for doc ordering and encdecpool control (start/stop). Also optimizes by deleting in batch and checking for a sequence in memory before asking the local staging for documents. --- client/changes/next-changelog.rst | 1 + client/src/leap/soledad/client/encdecpool.py | 183 ++++++++------------- .../src/leap/soledad/client/http_target/fetch.py | 9 +- client/src/leap/soledad/client/sqlcipher.py | 2 +- .../leap/soledad/common/tests/test_encdecpool.py | 47 +++++- .../src/leap/soledad/common/tests/test_server.py | 4 +- 6 files changed, 122 insertions(+), 124 deletions(-) diff --git a/client/changes/next-changelog.rst b/client/changes/next-changelog.rst index a696fe10..c676625f 100644 --- a/client/changes/next-changelog.rst +++ b/client/changes/next-changelog.rst @@ -26,6 +26,7 @@ Misc - Refactor bootstrap to remove shared db lock. - `#1236 `_: Description of the new feature corresponding with issue #1236. - Some change without issue number. +- Removed multiprocessing from encdecpool with some extra refactoring. Known Issues ~~~~~~~~~~~~ diff --git a/client/src/leap/soledad/client/encdecpool.py b/client/src/leap/soledad/client/encdecpool.py index 576b8b2c..218ebfa9 100644 --- a/client/src/leap/soledad/client/encdecpool.py +++ b/client/src/leap/soledad/client/encdecpool.py @@ -25,7 +25,7 @@ during synchronization. import json import logging -from twisted.internet import reactor +from twisted.internet.task import LoopingCall from twisted.internet import threads from twisted.internet import defer from twisted.python import log @@ -167,26 +167,14 @@ class SyncEncrypterPool(SyncEncryptDecryptPool): SyncEncryptDecryptPool.stop(self) - def enqueue_doc_for_encryption(self, doc): + def encrypt_doc(self, doc): """ - Enqueue a document for encryption. + Encrypt document asynchronously then insert it on + local staging database. :param doc: The document to be encrypted. :type doc: SoledadDocument """ - self._encrypt_doc(doc) - - def _encrypt_doc(self, doc): - """ - Symmetrically encrypt a document. - - :param doc: The document with contents to be encrypted. - :type doc: SoledadDocument - - :param workers: Whether to defer the decryption to the multiprocess - pool of workers. Useful for debugging purposes. - :type workers: bool - """ soledad_assert(self._crypto is not None, "need a crypto object") docstr = doc.get_json() key = self._crypto.doc_passphrase(doc.doc_id) @@ -276,8 +264,8 @@ def decrypt_doc_task(doc_id, doc_rev, content, gen, trans_id, key, secret, :type doc_id: str :param doc_rev: The document revision. :type doc_rev: str - :param content: The encrypted content of the document. - :type content: str + :param content: The encrypted content of the document as JSON dict. + :type content: dict :param gen: The generation corresponding to the modification of that document. :type gen: int @@ -294,7 +282,6 @@ def decrypt_doc_task(doc_id, doc_rev, content, gen, trans_id, key, secret, :return: A tuple containing the doc id, revision and encrypted content. :rtype: tuple(str, str, str) """ - content = json.loads(content) if type(content) == str else content decrypted_content = decrypt_doc_dict(content, doc_id, doc_rev, key, secret) return doc_id, doc_rev, decrypted_content, gen, trans_id, idx @@ -356,14 +343,12 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): self._processed_docs = 0 self._last_inserted_idx = 0 - # a list that holds the asynchronous decryption results so they can be - # collected when they are ready - self._async_results = [] - # initialize db and make sure any database operation happens after # db initialization self._deferred_init = self._init_db() self._wait_init_db('_runOperation', '_runQuery') + self._loop = LoopingCall(self._decrypt_and_recurse) + self._decrypted_docs_indexes = set() def _wait_init_db(self, *methods): """ @@ -408,11 +393,13 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): SyncEncryptDecryptPool.start(self) self._docs_to_process = docs_to_process self._deferred = defer.Deferred() - reactor.callWhenRunning(self._launch_decrypt_and_recurse) + self._loop.start(self.DECRYPT_LOOP_PERIOD) - def _launch_decrypt_and_recurse(self): - d = self._decrypt_and_recurse() - d.addErrback(self._errback) + def stop(self): + if self._loop.running: + self._loop.stop() + self._finish() + SyncEncryptDecryptPool.stop(self) def _errback(self, failure): log.err(failure) @@ -431,8 +418,8 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): def insert_encrypted_received_doc( self, doc_id, doc_rev, content, gen, trans_id, idx): """ - Insert a received message with encrypted content, to be decrypted later - on. + Decrypt and insert a received document into local staging area to be + processed later on. :param doc_id: The document ID. :type doc_id: str @@ -447,11 +434,19 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): :param idx: The index of this document in the current sync process. :type idx: int - :return: A deferred that will fire when the operation in the database - has finished. + :return: A deferred that will fire after the decrypted document has + been inserted in the sync db. :rtype: twisted.internet.defer.Deferred """ - return self._async_decrypt_doc(doc_id, doc_rev, content, gen, trans_id, idx) + soledad_assert(self._crypto is not None, "need a crypto object") + + key = self._crypto.doc_passphrase(doc_id) + secret = self._crypto.secret + args = doc_id, doc_rev, content, gen, trans_id, key, secret, idx + # decrypt asynchronously + doc = decrypt_doc_task(*args) + # callback will insert it for later processing + return self._decrypt_doc_cb(doc) def insert_received_doc( self, doc_id, doc_rev, content, gen, trans_id, idx): @@ -481,52 +476,26 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): content = json.dumps(content) query = "INSERT OR REPLACE INTO '%s' VALUES (?, ?, ?, ?, ?, ?, ?)" \ % self.TABLE_NAME - return self._runOperation( + d = self._runOperation( query, (doc_id, doc_rev, content, gen, trans_id, 0, idx)) + d.addCallback(lambda _: self._decrypted_docs_indexes.add(idx)) + return d - def _delete_received_doc(self, doc_id): + def _delete_received_docs(self, doc_ids): """ - Delete a received doc after it was inserted into the local db. + Delete a list of received docs after get them inserted into the db. - :param doc_id: Document ID. - :type doc_id: str + :param doc_id: Document ID list. + :type doc_id: list :return: A deferred that will fire when the operation in the database has finished. :rtype: twisted.internet.defer.Deferred """ - query = "DELETE FROM '%s' WHERE doc_id=?" \ - % self.TABLE_NAME - return self._runOperation(query, (doc_id,)) - - def _async_decrypt_doc(self, doc_id, rev, content, gen, trans_id, idx): - """ - Dispatch an asynchronous document decrypting routine and save the - result object. - - :param doc_id: The ID for the document with contents to be encrypted. - :type doc: str - :param rev: The revision of the document. - :type rev: str - :param content: The serialized content of the document. - :type content: str - :param gen: The generation corresponding to the modification of that - document. - :type gen: int - :param trans_id: The transaction id corresponding to the modification - of that document. - :type trans_id: str - :param idx: The index of this document in the current sync process. - :type idx: int - """ - soledad_assert(self._crypto is not None, "need a crypto object") - - key = self._crypto.doc_passphrase(doc_id) - secret = self._crypto.secret - args = doc_id, rev, content, gen, trans_id, key, secret, idx - # decrypt asynchronously - d = threads.deferToThread(decrypt_doc_task, *args) - d.addCallback(self._decrypt_doc_cb) + placeholders = ', '.join('?' for _ in doc_ids) + query = "DELETE FROM '%s' WHERE doc_id in (%s)" \ + % (self.TABLE_NAME, placeholders) + return self._runOperation(query, (doc_ids)) def _decrypt_doc_cb(self, result): """ @@ -547,7 +516,8 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): return self.insert_received_doc( doc_id, rev, content, gen, trans_id, idx) - def _get_docs(self, encrypted=None, order_by='idx', order='ASC'): + def _get_docs(self, encrypted=None, order_by='idx', order='ASC', + sequence=None): """ Get documents from the received docs table in the sync db. @@ -565,8 +535,13 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): """ query = "SELECT doc_id, rev, content, gen, trans_id, encrypted, " \ "idx FROM %s" % self.TABLE_NAME - if encrypted is not None: - query += " WHERE encrypted = %d" % int(encrypted) + if encrypted or sequence: + query += " WHERE" + if encrypted: + query += " encrypted = %d" % int(encrypted) + if sequence: + query += " idx in (" + ', '.join(sequence) + ")" + query += " ORDER BY %s %s" % (order_by, order) return self._runQuery(query) @@ -579,18 +554,19 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): documents. :rtype: twisted.internet.defer.Deferred """ - # here, we fetch the list of decrypted documents and compare with the - # index of the last succesfully processed document. - decrypted_docs = yield self._get_docs(encrypted=False) - insertable = [] - last_idx = self._last_inserted_idx - for doc_id, rev, content, gen, trans_id, encrypted, idx in \ - decrypted_docs: - if (idx != last_idx + 1): - break - insertable.append((doc_id, rev, content, gen, trans_id, idx)) - last_idx += 1 - defer.returnValue(insertable) + # Here, check in memory what are the insertable indexes that can + # form a sequence starting from the last inserted index + sequence = [] + insertable_docs = [] + next_index = self._last_inserted_idx + 1 + while next_index in self._decrypted_docs_indexes: + sequence.append(str(next_index)) + next_index += 1 + # Then fetch all the ones ready for insertion. + if sequence: + insertable_docs = yield self._get_docs(encrypted=False, + sequence=sequence) + defer.returnValue(insertable_docs) @defer.inlineCallbacks def _process_decrypted_docs(self): @@ -603,36 +579,18 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): :rtype: twisted.internet.defer.Deferred """ insertable = yield self._get_insertable_docs() + processed_docs_ids = [] for doc_fields in insertable: method = self._insert_decrypted_local_doc # FIXME: This is used only because SQLCipherU1DBSync is synchronous # When adbapi is used there is no need for an external thread # Without this the reactor can freeze and fail docs download yield threads.deferToThread(method, *doc_fields) - defer.returnValue(insertable) - - def _delete_processed_docs(self, inserted): - """ - Delete from the sync db documents that have been processed. - - :param inserted: List of documents inserted in the previous process - step. - :type inserted: list - - :return: A list of deferreds that will fire when each operation in the - database has finished. - :rtype: twisted.internet.defer.DeferredList - """ - deferreds = [] - for doc_id, doc_rev, _, _, _, _ in inserted: - deferreds.append( - self._delete_received_doc(doc_id)) - if not deferreds: - return defer.succeed(None) - return defer.gatherResults(deferreds) + processed_docs_ids.append(doc_fields[0]) + yield self._delete_received_docs(processed_docs_ids) def _insert_decrypted_local_doc(self, doc_id, doc_rev, content, - gen, trans_id, idx): + gen, trans_id, encrypted, idx): """ Insert the decrypted document into the local replica. @@ -693,20 +651,19 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): delete operations have been executed. :rtype: twisted.internet.defer.Deferred """ + if not self.running: + defer.returnValue(None) processed = self._processed_docs pending = self._docs_to_process if processed < pending: - docs = yield self._process_decrypted_docs() - yield self._delete_processed_docs(docs) - # recurse - self._delayed_call = reactor.callLater( - self.DECRYPT_LOOP_PERIOD, - self._launch_decrypt_and_recurse) + yield self._process_decrypted_docs() else: self._finish() def _finish(self): self._processed_docs = 0 self._last_inserted_idx = 0 - self._deferred.callback(None) + self._decrypted_docs_indexes = set() + if not self._deferred.called: + self._deferred.callback(None) diff --git a/client/src/leap/soledad/client/http_target/fetch.py b/client/src/leap/soledad/client/http_target/fetch.py index fda90909..9f7a4193 100644 --- a/client/src/leap/soledad/client/http_target/fetch.py +++ b/client/src/leap/soledad/client/http_target/fetch.py @@ -19,7 +19,6 @@ import json from u1db import errors from u1db.remote import utils from twisted.internet import defer -from twisted.internet import threads from leap.soledad.common.document import SoledadDocument from leap.soledad.client.events import SOLEDAD_SYNC_RECEIVE_STATUS from leap.soledad.client.events import emit_async @@ -76,7 +75,7 @@ class HTTPDocFetcher(object): last_known_generation, last_known_trans_id, sync_id, 0) self._received_docs = 0 - number_of_changes, ngen, ntrans = yield self._insert_received_doc(doc, 1, 1) + number_of_changes, ngen, ntrans = self._insert_received_doc(doc, 1, 1) if ngen: new_generation = ngen @@ -138,7 +137,6 @@ class HTTPDocFetcher(object): body=str(body), content_type='application/x-soledad-sync-get') - @defer.inlineCallbacks def _insert_received_doc(self, response, idx, total): """ Insert a received document into the local replica. @@ -152,8 +150,7 @@ class HTTPDocFetcher(object): """ new_generation, new_transaction_id, number_of_changes, doc_id, \ rev, content, gen, trans_id = \ - (yield threads.deferToThread(self._parse_received_doc_response, - response)) + self._parse_received_doc_response(response) if doc_id is not None: # decrypt incoming document and insert into local database # ------------------------------------------------------------- @@ -188,7 +185,7 @@ class HTTPDocFetcher(object): self._received_docs += 1 user_data = {'uuid': self.uuid, 'userid': self.userid} _emit_receive_status(user_data, self._received_docs, total) - defer.returnValue((number_of_changes, new_generation, new_transaction_id)) + return number_of_changes, new_generation, new_transaction_id def _parse_received_doc_response(self, response): """ diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index 22ddc87d..cdc7255c 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -278,7 +278,7 @@ class SQLCipherDatabase(sqlite_backend.SQLitePartialExpandDatabase): doc_rev = sqlite_backend.SQLitePartialExpandDatabase.put_doc(self, doc) if self.defer_encryption: # TODO move to api? - self._sync_enc_pool.enqueue_doc_for_encryption(doc) + self._sync_enc_pool.encrypt_doc(doc) return doc_rev # diff --git a/common/src/leap/soledad/common/tests/test_encdecpool.py b/common/src/leap/soledad/common/tests/test_encdecpool.py index 694eb7ad..6676c298 100644 --- a/common/src/leap/soledad/common/tests/test_encdecpool.py +++ b/common/src/leap/soledad/common/tests/test_encdecpool.py @@ -57,13 +57,13 @@ class TestSyncEncrypterPool(BaseSoledadTest): self.assertIsNone(doc) @inlineCallbacks - def test_enqueue_doc_for_encryption_and_get_encrypted_doc(self): + def test_encrypt_doc_and_get_it_back(self): """ Test that the pool actually encrypts a document added to the queue. """ doc = SoledadDocument( doc_id=DOC_ID, rev=DOC_REV, json=json.dumps(DOC_CONTENT)) - self._pool.enqueue_doc_for_encryption(doc) + self._pool.encrypt_doc(doc) # exhaustivelly attempt to get the encrypted document encrypted = None @@ -117,6 +117,16 @@ class TestSyncDecrypterPool(BaseSoledadTest): self._pool.deferred.addCallback(_assert_doc_was_inserted) return self._pool.deferred + def test_looping_control(self): + """ + Start and stop cleanly. + """ + self._pool.start(10) + self.assertTrue(self._pool.running) + self._pool.stop() + self.assertFalse(self._pool.running) + self.assertTrue(self._pool.deferred.called) + def test_insert_received_doc_many(self): """ Test that many documents added to the pool are inserted using the @@ -179,6 +189,38 @@ class TestSyncDecrypterPool(BaseSoledadTest): _assert_doc_was_decrypted_and_inserted) return self._pool.deferred + @inlineCallbacks + def test_processing_order(self): + """ + This test ensures that processing of documents only occur if there is + a sequence in place. + """ + crypto = self._soledad._crypto + docs = [] + for i in xrange(1, 10): + i = str(i) + doc = SoledadDocument( + doc_id=DOC_ID + i, rev=DOC_REV + i, + json=json.dumps(DOC_CONTENT)) + encrypted_content = json.loads(crypto.encrypt_doc(doc)) + docs.append((doc, encrypted_content)) + + # insert the encrypted document in the pool + self._pool.start(10) # pool is expecting to process 10 docs + # first three arrives, forming a sequence + for i, (doc, encrypted_content) in enumerate(docs[:3]): + gen = idx = i + 1 + yield self._pool.insert_encrypted_received_doc( + doc.doc_id, doc.rev, encrypted_content, gen, "trans_id", idx) + # last one arrives alone, so it can't be processed + doc, encrypted_content = docs[-1] + yield self._pool.insert_encrypted_received_doc( + doc.doc_id, doc.rev, encrypted_content, 10, "trans_id", 10) + + yield self._pool._decrypt_and_recurse() + + self.assertEqual(3, self._pool._processed_docs) + def test_insert_encrypted_received_doc_many(self, many=100): """ Test that many encrypted documents added to the pool are decrypted and @@ -241,3 +283,4 @@ class TestSyncDecrypterPool(BaseSoledadTest): decrypted_docs = yield self._pool._get_docs(encrypted=False) # check that decrypted docs staging is clean self.assertEquals([], decrypted_docs) + self._pool.stop() diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py index bf6c1515..ba7edfe3 100644 --- a/common/src/leap/soledad/common/tests/test_server.py +++ b/common/src/leap/soledad/common/tests/test_server.py @@ -481,8 +481,8 @@ class EncryptedSyncTestCase( Test if Soledad can sync very large files. """ self.skipTest( - "Work in progress. For reference, see: " - "https://leap.se/code/issues/7370") + "Work in progress. For reference, see: " + "https://leap.se/code/issues/7370") length = 100 * (10 ** 6) # 100 MB return self._test_encrypted_sym_sync(doc_size=length, number_of_docs=1) -- cgit v1.2.3 From ebcf2a098fb8e9b1211e31b4955aa67cfebc5854 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 16 Feb 2016 12:36:43 -0300 Subject: [bug] delete all docs on start and ensure isolation Docs created from one failed sync would be there for the next one, possibly causing a lot of hard to find errors. This commit adds a sync_id field to track each sync documents isolated and cleans up the pool on start instead of constructor. --- client/src/leap/soledad/client/encdecpool.py | 96 +++++++--------------- .../src/leap/soledad/client/http_target/fetch.py | 7 +- 2 files changed, 35 insertions(+), 68 deletions(-) diff --git a/client/src/leap/soledad/client/encdecpool.py b/client/src/leap/soledad/client/encdecpool.py index 218ebfa9..7d646c51 100644 --- a/client/src/leap/soledad/client/encdecpool.py +++ b/client/src/leap/soledad/client/encdecpool.py @@ -24,6 +24,7 @@ during synchronization. import json import logging +from uuid import uuid4 from twisted.internet.task import LoopingCall from twisted.internet import threads @@ -65,13 +66,9 @@ class SyncEncryptDecryptPool(object): self._started = False def start(self): - if self.running: - return self._started = True def stop(self): - if not self.running: - return self._started = False # maybe cancel the next delayed call if self._delayed_call \ @@ -312,7 +309,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): """ TABLE_NAME = "docs_received" FIELD_NAMES = "doc_id PRIMARY KEY, rev, content, gen, " \ - "trans_id, encrypted, idx" + "trans_id, encrypted, idx, sync_id" """ Period of recurrence of the periodic decrypting task, in seconds. @@ -343,42 +340,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): self._processed_docs = 0 self._last_inserted_idx = 0 - # initialize db and make sure any database operation happens after - # db initialization - self._deferred_init = self._init_db() - self._wait_init_db('_runOperation', '_runQuery') self._loop = LoopingCall(self._decrypt_and_recurse) - self._decrypted_docs_indexes = set() - - def _wait_init_db(self, *methods): - """ - Methods that need to wait for db initialization. - - :param methods: methods that need to wait for initialization - :type methods: tuple(str) - """ - self._waiting = [] - self._stored = {} - - def _restore(_): - for method in self._stored: - setattr(self, method, self._stored[method]) - for d in self._waiting: - d.callback(None) - - def _makeWrapper(method): - def wrapper(*args, **kw): - d = defer.Deferred() - d.addCallback(lambda _: self._stored[method](*args, **kw)) - self._waiting.append(d) - return d - return wrapper - - for method in methods: - self._stored[method] = getattr(self, method) - setattr(self, method, _makeWrapper(method)) - - self._deferred_init.addCallback(_restore) def start(self, docs_to_process): """ @@ -391,9 +353,13 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): :type docs_to_process: int """ SyncEncryptDecryptPool.start(self) + self._decrypted_docs_indexes = set() + self._sync_id = uuid4().hex self._docs_to_process = docs_to_process self._deferred = defer.Deferred() - self._loop.start(self.DECRYPT_LOOP_PERIOD) + d = self._init_db() + d.addCallback(lambda _: self._loop.start(self.DECRYPT_LOOP_PERIOD)) + return d def stop(self): if self._loop.running: @@ -401,6 +367,17 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): self._finish() SyncEncryptDecryptPool.stop(self) + def _init_db(self): + """ + Empty the received docs table of the sync database. + + :return: A deferred that will fire when the operation in the database + has finished. + :rtype: twisted.internet.defer.Deferred + """ + query = "DELETE FROM %s WHERE sync_id <> ?" % (self.TABLE_NAME,) + return self._runOperation(query, (self._sync_id,)) + def _errback(self, failure): log.err(failure) self._deferred.errback(failure) @@ -474,10 +451,11 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): """ if not isinstance(content, str): content = json.dumps(content) - query = "INSERT OR REPLACE INTO '%s' VALUES (?, ?, ?, ?, ?, ?, ?)" \ + query = "INSERT OR REPLACE INTO '%s' VALUES (?, ?, ?, ?, ?, ?, ?, ?)" \ % self.TABLE_NAME d = self._runOperation( - query, (doc_id, doc_rev, content, gen, trans_id, 0, idx)) + query, (doc_id, doc_rev, content, gen, trans_id, 0, + idx, self._sync_id)) d.addCallback(lambda _: self._decrypted_docs_indexes.add(idx)) return d @@ -516,8 +494,7 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): return self.insert_received_doc( doc_id, rev, content, gen, trans_id, idx) - def _get_docs(self, encrypted=None, order_by='idx', order='ASC', - sequence=None): + def _get_docs(self, encrypted=None, sequence=None): """ Get documents from the received docs table in the sync db. @@ -525,9 +502,6 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): field equal to given parameter. :type encrypted: bool or None :param order_by: The name of the field to order results. - :type order_by: str - :param order: Whether the order should be ASC or DESC. - :type order: str :return: A deferred that will fire with the results of the database query. @@ -535,15 +509,18 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): """ query = "SELECT doc_id, rev, content, gen, trans_id, encrypted, " \ "idx FROM %s" % self.TABLE_NAME + parameters = [] if encrypted or sequence: - query += " WHERE" + query += " WHERE sync_id = ? and" + parameters += [self._sync_id] if encrypted: - query += " encrypted = %d" % int(encrypted) + query += " encrypted = ?" + parameters += [int(encrypted)] if sequence: - query += " idx in (" + ', '.join(sequence) + ")" - - query += " ORDER BY %s %s" % (order_by, order) - return self._runQuery(query) + query += " idx in (" + ', '.join('?' * len(sequence)) + ")" + parameters += [int(i) for i in sequence] + query += " ORDER BY idx ASC" + return self._runQuery(query, parameters) @defer.inlineCallbacks def _get_insertable_docs(self): @@ -625,17 +602,6 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): self._last_inserted_idx = idx self._processed_docs += 1 - def _init_db(self): - """ - Empty the received docs table of the sync database. - - :return: A deferred that will fire when the operation in the database - has finished. - :rtype: twisted.internet.defer.Deferred - """ - query = "DELETE FROM %s WHERE 1" % (self.TABLE_NAME,) - return self._runOperation(query) - @defer.inlineCallbacks def _decrypt_and_recurse(self): """ diff --git a/client/src/leap/soledad/client/http_target/fetch.py b/client/src/leap/soledad/client/http_target/fetch.py index 9f7a4193..9801c3d9 100644 --- a/client/src/leap/soledad/client/http_target/fetch.py +++ b/client/src/leap/soledad/client/http_target/fetch.py @@ -81,9 +81,6 @@ class HTTPDocFetcher(object): new_generation = ngen new_transaction_id = ntrans - if defer_decryption: - self._sync_decr_pool.start(number_of_changes) - # --------------------------------------------------------------------- # maybe receive the rest of the documents # --------------------------------------------------------------------- @@ -151,6 +148,10 @@ class HTTPDocFetcher(object): new_generation, new_transaction_id, number_of_changes, doc_id, \ rev, content, gen, trans_id = \ self._parse_received_doc_response(response) + + if self._sync_decr_pool and not self._sync_decr_pool.running: + self._sync_decr_pool.start(number_of_changes) + if doc_id is not None: # decrypt incoming document and insert into local database # ------------------------------------------------------------- -- cgit v1.2.3 From f6a7cdded4285af2335263a058479fa158980b31 Mon Sep 17 00:00:00 2001 From: NavaL Date: Fri, 29 Apr 2016 18:47:13 +0200 Subject: [bug] ensures docs_received table has the sync_id column For the case where the user already has data synced, this commit will migrate the docs_received table to have the column sync_id. That is required by the refactoring in the previous commits. --- client/src/leap/soledad/client/encdecpool.py | 12 ++++++-- .../leap/soledad/common/tests/test_encdecpool.py | 34 ++++++++++++++++++---- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/client/src/leap/soledad/client/encdecpool.py b/client/src/leap/soledad/client/encdecpool.py index 7d646c51..e348f545 100644 --- a/client/src/leap/soledad/client/encdecpool.py +++ b/client/src/leap/soledad/client/encdecpool.py @@ -369,14 +369,22 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): def _init_db(self): """ + Ensure sync_id column is present then Empty the received docs table of the sync database. :return: A deferred that will fire when the operation in the database has finished. :rtype: twisted.internet.defer.Deferred """ - query = "DELETE FROM %s WHERE sync_id <> ?" % (self.TABLE_NAME,) - return self._runOperation(query, (self._sync_id,)) + ensure_sync_id_column = "ALTER TABLE %s ADD COLUMN sync_id" % self.TABLE_NAME + d = self._runQuery(ensure_sync_id_column) + + def empty_received_docs(_): + query = "DELETE FROM %s WHERE sync_id <> ?" % (self.TABLE_NAME,) + return self._runOperation(query, (self._sync_id,)) + + d.addCallbacks(empty_received_docs, empty_received_docs) + return d def _errback(self, failure): log.err(failure) diff --git a/common/src/leap/soledad/common/tests/test_encdecpool.py b/common/src/leap/soledad/common/tests/test_encdecpool.py index 6676c298..9d98f44d 100644 --- a/common/src/leap/soledad/common/tests/test_encdecpool.py +++ b/common/src/leap/soledad/common/tests/test_encdecpool.py @@ -20,6 +20,7 @@ Tests for encryption and decryption pool. import json from random import shuffle +from mock import MagicMock from twisted.internet.defer import inlineCallbacks from leap.soledad.client.encdecpool import SyncEncrypterPool @@ -27,7 +28,7 @@ from leap.soledad.client.encdecpool import SyncDecrypterPool from leap.soledad.common.document import SoledadDocument from leap.soledad.common.tests.util import BaseSoledadTest - +from twisted.internet import defer DOC_ID = "mydoc" DOC_REV = "rev" @@ -84,14 +85,18 @@ class TestSyncDecrypterPool(BaseSoledadTest): """ self._inserted_docs.append((doc, gen, trans_id)) - def setUp(self): - BaseSoledadTest.setUp(self) - # setup the pool - self._pool = SyncDecrypterPool( + def _setup_pool(self, sync_db=None): + sync_db = sync_db or self._soledad._sync_db + return SyncDecrypterPool( self._soledad._crypto, - self._soledad._sync_db, + sync_db, source_replica_uid=self._soledad._dbpool.replica_uid, insert_doc_cb=self._insert_doc_cb) + + def setUp(self): + BaseSoledadTest.setUp(self) + # setup the pool + self._pool = self._setup_pool() # reset the inserted docs mock self._inserted_docs = [] @@ -127,6 +132,23 @@ class TestSyncDecrypterPool(BaseSoledadTest): self.assertFalse(self._pool.running) self.assertTrue(self._pool.deferred.called) + def test_sync_id_column_is_created_if_non_existing_in_docs_received_table(self): + """ + Test that docs_received table is migrated, and has the sync_id column + """ + mock_run_query = MagicMock(return_value=defer.succeed(None)) + mock_sync_db = MagicMock() + mock_sync_db.runQuery = mock_run_query + pool = self._setup_pool(mock_sync_db) + d = pool.start(10) + pool.stop() + + def assert_trial_to_create_sync_id_column(_): + mock_run_query.assert_called_once_with("ALTER TABLE docs_received ADD COLUMN sync_id") + + d.addCallback(assert_trial_to_create_sync_id_column) + return d + def test_insert_received_doc_many(self): """ Test that many documents added to the pool are inserted using the -- cgit v1.2.3 From 86b74a3404c3dc98b422d348edc65848b381f5f1 Mon Sep 17 00:00:00 2001 From: drebs Date: Sun, 1 May 2016 19:57:42 -0300 Subject: [feature] add sync phase stats --- client/changes/next-changelog.rst | 2 ++ client/src/leap/soledad/client/api.py | 17 +++++++++ .../leap/soledad/client/http_target/__init__.py | 10 ++++++ client/src/leap/soledad/client/http_target/api.py | 22 ++++++++++++ client/src/leap/soledad/client/sqlcipher.py | 13 +++++++ client/src/leap/soledad/client/sync.py | 42 ++++++++++++++++++++++ 6 files changed, 106 insertions(+) diff --git a/client/changes/next-changelog.rst b/client/changes/next-changelog.rst index c676625f..cffe4954 100644 --- a/client/changes/next-changelog.rst +++ b/client/changes/next-changelog.rst @@ -23,6 +23,8 @@ Bugfixes Misc ~~~~ +- Add ability to get information about sync phases for profiling purposes. +- Add script for setting up develop environment. - Refactor bootstrap to remove shared db lock. - `#1236 `_: Description of the new feature corresponding with issue #1236. - Some change without issue number. diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index d83291e7..a1588aa9 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -62,6 +62,13 @@ from leap.soledad.client import encdecpool logger = logging.getLogger(name=__name__) + +# we may want to collect statistics from the sync process +DO_STATS = False +if os.environ.get('SOLEDAD_STATS'): + DO_STATS = True + + # # Constants # @@ -297,6 +304,16 @@ class Soledad(object): sync_db=self._sync_db, sync_enc_pool=self._sync_enc_pool) + def sync_stats(self): + sync_phase = 0 + if getattr(self._dbsyncer, 'sync_phase', None): + sync_phase = self._dbsyncer.sync_phase[0] + sync_exchange_phase = 0 + if getattr(self._dbsyncer, 'syncer', None): + if getattr(self._dbsyncer.syncer, 'sync_exchange_phase', None): + sync_exchange_phase = self._dbsyncer.syncer.sync_exchange_phase[0] + return sync_phase, sync_exchange_phase + # # Closing methods # diff --git a/client/src/leap/soledad/client/http_target/__init__.py b/client/src/leap/soledad/client/http_target/__init__.py index a16531ef..b7e54aa4 100644 --- a/client/src/leap/soledad/client/http_target/__init__.py +++ b/client/src/leap/soledad/client/http_target/__init__.py @@ -22,6 +22,7 @@ after receiving. """ +import os import logging from leap.common.http import HTTPClient @@ -33,6 +34,12 @@ from leap.soledad.client.http_target.fetch import HTTPDocFetcher logger = logging.getLogger(__name__) +# we may want to collect statistics from the sync process +DO_STATS = False +if os.environ.get('SOLEDAD_STATS'): + DO_STATS = True + + class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher): """ @@ -93,3 +100,6 @@ class SoledadHTTPSyncTarget(SyncTargetAPI, HTTPDocSender, HTTPDocFetcher): # the duplicated syncing bug. This could be reduced to the 30s default # after implementing Cancellable Sync. See #7382 self._http = HTTPClient(cert_file, timeout=90) + + if DO_STATS: + self.sync_exchange_phase = [0] diff --git a/client/src/leap/soledad/client/http_target/api.py b/client/src/leap/soledad/client/http_target/api.py index 94354092..b19ce9ce 100644 --- a/client/src/leap/soledad/client/http_target/api.py +++ b/client/src/leap/soledad/client/http_target/api.py @@ -14,6 +14,8 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import os +import time import json import base64 @@ -27,6 +29,12 @@ from leap.soledad.common.errors import InvalidAuthTokenError from leap.soledad.client.http_target.support import readBody +# we may want to collect statistics from the sync process +DO_STATS = False +if os.environ.get('SOLEDAD_STATS'): + DO_STATS = True + + class SyncTargetAPI(SyncTarget): """ Declares public methods and implements u1db.SyncTarget. @@ -187,6 +195,10 @@ class SyncTargetAPI(SyncTarget): transaction id of the target replica. :rtype: twisted.internet.defer.Deferred """ + # ---------- phase 1: send docs to server ---------------------------- + if DO_STATS: + self.sync_exchange_phase[0] += 1 + # -------------------------------------------------------------------- self._ensure_callback = ensure_callback @@ -203,6 +215,11 @@ class SyncTargetAPI(SyncTarget): last_known_trans_id, sync_id) + # ---------- phase 2: receive docs ----------------------------------- + if DO_STATS: + self.sync_exchange_phase[0] += 1 + # -------------------------------------------------------------------- + cur_target_gen, cur_target_trans_id = yield self._receive_docs( last_known_generation, last_known_trans_id, ensure_callback, sync_id, @@ -214,6 +231,11 @@ class SyncTargetAPI(SyncTarget): cur_target_gen = gen_after_send cur_target_trans_id = trans_id_after_send + # ---------- phase 3: sync exchange is over -------------------------- + if DO_STATS: + self.sync_exchange_phase[0] += 1 + # -------------------------------------------------------------------- + defer.returnValue([cur_target_gen, cur_target_trans_id]) diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index cdc7255c..99f5dad8 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -72,6 +72,12 @@ logger = logging.getLogger(__name__) sqlite_backend.dbapi2 = sqlcipher_dbapi2 +# we may want to collect statistics from the sync process +DO_STATS = False +if os.environ.get('SOLEDAD_STATS'): + DO_STATS = True + + def initialize_sqlcipher_db(opts, on_init=None, check_same_thread=True): """ Initialize a SQLCipher database. @@ -453,6 +459,9 @@ class SQLCipherU1DBSync(SQLCipherDatabase): self.shutdownID = None + if DO_STATS: + self.sync_phase = None + @property def _replica_uid(self): return str(self.__replica_uid) @@ -497,6 +506,10 @@ class SQLCipherU1DBSync(SQLCipherDatabase): :rtype: Deferred """ syncer = self._get_syncer(url, creds=creds) + if DO_STATS: + self.sync_phase = syncer.sync_phase + self.syncer = syncer + self.sync_exchange_phase = syncer.sync_exchange_phase local_gen_before_sync = yield syncer.sync( defer_decryption=defer_decryption) self.received_docs = syncer.received_docs diff --git a/client/src/leap/soledad/client/sync.py b/client/src/leap/soledad/client/sync.py index 1879031f..9cafe62f 100644 --- a/client/src/leap/soledad/client/sync.py +++ b/client/src/leap/soledad/client/sync.py @@ -17,6 +17,8 @@ """ Soledad synchronization utilities. """ +import os +import time import logging from twisted.internet import defer @@ -29,6 +31,12 @@ from u1db.sync import Synchronizer logger = logging.getLogger(__name__) +# we may want to collect statistics from the sync process +DO_STATS = False +if os.environ.get('SOLEDAD_STATS'): + DO_STATS = True + + class SoledadSynchronizer(Synchronizer): """ Collect the state around synchronizing 2 U1DB replicas. @@ -42,6 +50,12 @@ class SoledadSynchronizer(Synchronizer): """ received_docs = [] + def __init__(self, *args, **kwargs): + Synchronizer.__init__(self, *args, **kwargs) + if DO_STATS: + self.sync_phase = [0] + self.sync_exchange_phase = None + @defer.inlineCallbacks def sync(self, defer_decryption=True): """ @@ -64,9 +78,16 @@ class SoledadSynchronizer(Synchronizer): the local generation before the synchronization was performed. :rtype: twisted.internet.defer.Deferred """ + sync_target = self.sync_target self.received_docs = [] + # ---------- phase 1: get sync info from server ---------------------- + if DO_STATS: + self.sync_phase[0] += 1 + self.sync_exchange_phase = self.sync_target.sync_exchange_phase + # -------------------------------------------------------------------- + # get target identifier, its current generation, # and its last-seen database generation for this source ensure_callback = None @@ -106,6 +127,11 @@ class SoledadSynchronizer(Synchronizer): self.source.validate_gen_and_trans_id( target_my_gen, target_my_trans_id) + # ---------- phase 2: what's changed --------------------------------- + if DO_STATS: + self.sync_phase[0] += 1 + # -------------------------------------------------------------------- + # what's changed since that generation and this current gen my_gen, _, changes = self.source.whats_changed(target_my_gen) logger.debug("Soledad sync: there are %d documents to send." @@ -130,6 +156,11 @@ class SoledadSynchronizer(Synchronizer): raise errors.InvalidTransactionId defer.returnValue(my_gen) + # ---------- phase 3: sync exchange ---------------------------------- + if DO_STATS: + self.sync_phase[0] += 1 + # -------------------------------------------------------------------- + # prepare to send all the changed docs changed_doc_ids = [doc_id for doc_id, _, _ in changes] docs_to_send = self.source.get_docs( @@ -162,6 +193,12 @@ class SoledadSynchronizer(Synchronizer): "my_gen": my_gen } self._syncing_info = info + + # ---------- phase 4: complete sync ---------------------------------- + if DO_STATS: + self.sync_phase[0] += 1 + # -------------------------------------------------------------------- + yield self.complete_sync() _, _, changes = self.source.whats_changed(target_my_gen) @@ -170,6 +207,11 @@ class SoledadSynchronizer(Synchronizer): just_received = list(set(changed_doc_ids) - set(ids_sent)) self.received_docs = just_received + # ---------- phase 5: sync is over ----------------------------------- + if DO_STATS: + self.sync_phase[0] += 1 + # -------------------------------------------------------------------- + defer.returnValue(my_gen) def complete_sync(self): -- cgit v1.2.3 From 7b468680cc2ad09cf836da0095330d941a5ea7b9 Mon Sep 17 00:00:00 2001 From: drebs Date: Sat, 7 May 2016 17:09:29 -0300 Subject: [feat] add recovery doc format version --- client/changes/next-changelog.rst | 1 + client/src/leap/soledad/client/secrets.py | 91 ++++++++++++++++++++++++++----- 2 files changed, 79 insertions(+), 13 deletions(-) diff --git a/client/changes/next-changelog.rst b/client/changes/next-changelog.rst index cffe4954..7ddb3a57 100644 --- a/client/changes/next-changelog.rst +++ b/client/changes/next-changelog.rst @@ -10,6 +10,7 @@ I've added a new category `Misc` so we can track doc/style/packaging stuff. Features ~~~~~~~~ +- Add recovery document format version for future migrations. - Use DeferredLock instead of its locking cousin. - `#1234 `_: Description of the new feature corresponding with issue #1234. - New feature without related issue number. diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py index 714b2dfe..c35b881e 100644 --- a/client/src/leap/soledad/client/secrets.py +++ b/client/src/leap/soledad/client/secrets.py @@ -143,6 +143,8 @@ class SoledadSecrets(object): KDF_LENGTH_KEY = 'kdf_length' KDF_SCRYPT = 'scrypt' CIPHER_AES256 = 'aes256' + RECOVERY_DOC_VERSION_KEY = 'version' + RECOVERY_DOC_VERSION = 1 """ Keys used to access storage secrets in recovery documents. """ @@ -192,9 +194,15 @@ class SoledadSecrets(object): # STAGE 1 - verify if secrets exist locally try: logger.info("Trying to load secrets from local storage...") - self._load_secrets_from_local_file() + version = self._load_secrets_from_local_file() + # eventually migrate local and remote stored documents from old + # format version + if version < self.RECOVERY_DOC_VERSION: + self._store_secrets() + self._upload_crypto_secrets() logger.info("Found secrets in local storage.") return + except NoStorageSecret: logger.info("Could not find secrets in local storage.") @@ -204,7 +212,12 @@ class SoledadSecrets(object): # server. try: logger.info('Trying to fetch secrets from remote storage...') - self._download_crypto_secrets() + version = self._download_crypto_secrets() + self._store_secrets() + # eventually migrate remote stored document from old format + # version + if version < self.RECOVERY_DOC_VERSION: + self._upload_crypto_secrets() logger.info('Found secrets in remote storage.') return except NoStorageSecret: @@ -240,6 +253,9 @@ class SoledadSecrets(object): def _load_secrets_from_local_file(self): """ Load storage secrets from local file. + + :return version: The version of the locally stored recovery document. + :raise NoStorageSecret: Raised if there are no secrets available in local storage. """ @@ -251,13 +267,18 @@ class SoledadSecrets(object): content = None with open(self._secrets_path, 'r') as f: content = json.loads(f.read()) - _, active_secret = self._import_recovery_document(content) + _, active_secret, version = self._import_recovery_document(content) + self._maybe_set_active_secret(active_secret) + return version + def _download_crypto_secrets(self): """ Download crypto secrets. + :return version: The version of the remotelly stored recovery document. + :raise NoStorageSecret: Raised if there are no secrets available in remote storage. """ @@ -268,9 +289,10 @@ class SoledadSecrets(object): if doc is None: raise NoStorageSecret - _, active_secret = self._import_recovery_document(doc.content) + _, active_secret, version = self._import_recovery_document(doc.content) self._maybe_set_active_secret(active_secret) - self._store_secrets() # save new secrets in local file + + return version def _gen_crypto_secrets(self): """ @@ -325,7 +347,7 @@ class SoledadSecrets(object): """ Export the storage secrets. - A recovery document has the following structure: + Current format of recovery document has the following structure: { 'storage_secrets': { @@ -336,6 +358,7 @@ class SoledadSecrets(object): }, }, 'active_secret': '', + 'version': '', } Note that multiple storage secrets might be stored in one recovery @@ -353,13 +376,14 @@ class SoledadSecrets(object): data = { self.STORAGE_SECRETS_KEY: encrypted_secrets, self.ACTIVE_SECRET_KEY: self._secret_id, + self.RECOVERY_DOC_VERSION_KEY: self.RECOVERY_DOC_VERSION, } return data def _import_recovery_document(self, data): """ - Import storage secrets for symmetric encryption and uuid (if present) - from a recovery document. + Import storage secrets for symmetric encryption from a recovery + document. Note that this method does not store the imported data on disk. For that, use C{self._store_secrets()}. @@ -367,11 +391,44 @@ class SoledadSecrets(object): :param data: The recovery document. :type data: dict - :return: A tuple containing the number of imported secrets and the - secret_id of the last active secret. - :rtype: (int, str) + :return: A tuple containing the number of imported secrets, the + secret_id of the last active secret, and the recovery + document format version. + :rtype: (int, str, int) """ soledad_assert(self.STORAGE_SECRETS_KEY in data) + version = data.get(self.RECOVERY_DOC_VERSION_KEY, 1) + meth = getattr(self, '_import_recovery_document_version_%d' % version) + secret_count, active_secret = meth(data) + return secret_count, active_secret, version + + def _import_recovery_document_version_1(self, data): + """ + Import storage secrets for symmetric encryption from a recovery + document with format version 1. + + Version 1 of recovery document has the following structure: + + { + 'storage_secrets': { + '': { + 'cipher': 'aes256', + 'length': , + 'secret': '', + }, + }, + 'active_secret': '', + 'version': '', + } + + :param data: The recovery document. + :type data: dict + + :return: A tuple containing the number of imported secrets, the + secret_id of the last active secret, and the recovery + document format version. + :rtype: (int, str, int) + """ # include secrets in the secret pool. secret_count = 0 secrets = data[self.STORAGE_SECRETS_KEY].items() @@ -384,7 +441,7 @@ class SoledadSecrets(object): if secret_id not in self._secrets: try: self._secrets[secret_id] = \ - self._decrypt_storage_secret(encrypted_secret) + self._decrypt_storage_secret_version_1(encrypted_secret) secret_count += 1 except SecretsException as e: logger.error("Failed to decrypt storage secret: %s" @@ -443,13 +500,21 @@ class SoledadSecrets(object): # Management of secret for symmetric encryption. # - def _decrypt_storage_secret(self, encrypted_secret_dict): + def _decrypt_storage_secret_version_1(self, encrypted_secret_dict): """ Decrypt the storage secret. Storage secret is encrypted before being stored. This method decrypts and returns the decrypted storage secret. + Version 1 of storage secret format has the following structure: + + '': { + 'cipher': 'aes256', + 'length': , + 'secret': '', + }, + :param encrypted_secret_dict: The encrypted storage secret. :type encrypted_secret_dict: dict -- cgit v1.2.3 From ef7aabde344371882b38eacca7294727f6ccaca0 Mon Sep 17 00:00:00 2001 From: drebs Date: Sat, 7 May 2016 17:11:09 -0300 Subject: [test] turn test for _gen_secret into many unit tests --- .../src/leap/soledad/common/tests/test_crypto.py | 82 ++++++++++++---------- 1 file changed, 46 insertions(+), 36 deletions(-) diff --git a/common/src/leap/soledad/common/tests/test_crypto.py b/common/src/leap/soledad/common/tests/test_crypto.py index 4fc3161d..5ced024b 100644 --- a/common/src/leap/soledad/common/tests/test_crypto.py +++ b/common/src/leap/soledad/common/tests/test_crypto.py @@ -26,7 +26,6 @@ from leap.soledad.common.document import SoledadDocument from leap.soledad.common.tests.util import BaseSoledadTest from leap.soledad.common.crypto import WrongMacError from leap.soledad.common.crypto import UnknownMacMethodError -from leap.soledad.common.crypto import EncryptionMethods from leap.soledad.common.crypto import ENC_JSON_KEY from leap.soledad.common.crypto import ENC_SCHEME_KEY from leap.soledad.common.crypto import MAC_KEY @@ -67,7 +66,7 @@ class RecoveryDocumentTestCase(BaseSoledadTest): rd = self._soledad.secrets._export_recovery_document() secret_id = rd[self._soledad.secrets.STORAGE_SECRETS_KEY].items()[0][0] # assert exported secret is the same - secret = self._soledad.secrets._decrypt_storage_secret( + secret = self._soledad.secrets._decrypt_storage_secret_version_1( rd[self._soledad.secrets.STORAGE_SECRETS_KEY][secret_id]) self.assertEqual(secret_id, self._soledad.secrets._secret_id) self.assertEqual(secret, self._soledad.secrets._secrets[secret_id]) @@ -93,47 +92,58 @@ class RecoveryDocumentTestCase(BaseSoledadTest): class SoledadSecretsTestCase(BaseSoledadTest): - def test__gen_secret(self): - # instantiate and save secret_id - sol = self._soledad_instance(user='user@leap.se') - self.assertTrue(len(sol.secrets._secrets) == 1) - secret_id_1 = sol.secrets.secret_id - # assert id is hash of secret + def test_new_soledad_instance_generates_one_secret(self): self.assertTrue( - secret_id_1 == hashlib.sha256(sol.storage_secret).hexdigest()) + self._soledad.storage_secret is not None, + "Expected secret to be something different than None") + number_of_secrets = len(self._soledad.secrets._secrets) + self.assertTrue( + number_of_secrets == 1, + "Expected exactly 1 secret, got %d instead." % number_of_secrets) + + def test_generated_secret_is_of_correct_type(self): + expected_type = str + self.assertIsInstance( + self._soledad.storage_secret, expected_type, + "Expected secret to be of type %s" % expected_type) + + def test_generated_secret_has_correct_lengt(self): + expected_length = self._soledad.secrets.GEN_SECRET_LENGTH + actual_length = len(self._soledad.storage_secret) + self.assertTrue( + expected_length == actual_length, + "Expected secret with length %d, got %d instead." + % (expected_length, actual_length)) + + def test_generated_secret_id_is_sha256_hash_of_secret(self): + generated = self._soledad.secrets.secret_id + expected = hashlib.sha256(self._soledad.storage_secret).hexdigest() + self.assertTrue( + generated == expected, + "Expeceted generated secret id to be sha256 hash, got something " + "else instead.") + + def test_generate_new_secret_generates_different_secret_id(self): # generate new secret - secret_id_2 = sol.secrets._gen_secret() - self.assertTrue(secret_id_1 != secret_id_2) - sol.close() - # re-instantiate - sol = self._soledad_instance(user='user@leap.se') - sol.secrets.set_secret_id(secret_id_1) - # assert ids are valid - self.assertTrue(len(sol.secrets._secrets) == 2) - self.assertTrue(secret_id_1 in sol.secrets._secrets) - self.assertTrue(secret_id_2 in sol.secrets._secrets) - # assert format of secret 1 - self.assertTrue(sol.storage_secret is not None) - self.assertIsInstance(sol.storage_secret, str) - secret_length = sol.secrets.GEN_SECRET_LENGTH - self.assertTrue(len(sol.storage_secret) == secret_length) - # assert format of secret 2 - sol.secrets.set_secret_id(secret_id_2) - self.assertTrue(sol.storage_secret is not None) - self.assertIsInstance(sol.storage_secret, str) - self.assertTrue(len(sol.storage_secret) == secret_length) - # assert id is hash of new secret + secret_id_1 = self._soledad.secrets.secret_id + secret_id_2 = self._soledad.secrets._gen_secret() + self.assertTrue( + len(self._soledad.secrets._secrets) == 2, + "Expected exactly 2 secrets.") + self.assertTrue( + secret_id_1 != secret_id_2, + "Expected IDs of secrets to be distinct.") + self.assertTrue( + secret_id_1 in self._soledad.secrets._secrets, + "Expected to find ID of first secret in Soledad Secrets.") self.assertTrue( - secret_id_2 == hashlib.sha256(sol.storage_secret).hexdigest()) - sol.close() + secret_id_2 in self._soledad.secrets._secrets, + "Expected to find ID of second secret in Soledad Secrets.") def test__has_secret(self): - sol = self._soledad_instance( - user='user@leap.se', prefix=self.rand_prefix) self.assertTrue( - sol.secrets._has_secret(), + self._soledad._secrets._has_secret(), "Should have a secret at this point") - sol.close() class MacAuthTestCase(BaseSoledadTest): -- cgit v1.2.3 From 4a1b8f2467a73a9c9a80acf19383786aa188042b Mon Sep 17 00:00:00 2001 From: drebs Date: Sat, 7 May 2016 17:11:41 -0300 Subject: [test] use inline deferreds in test for change passphrase --- .../src/leap/soledad/common/tests/test_soledad.py | 40 ++++++++-------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/common/src/leap/soledad/common/tests/test_soledad.py b/common/src/leap/soledad/common/tests/test_soledad.py index b48915eb..aa52a733 100644 --- a/common/src/leap/soledad/common/tests/test_soledad.py +++ b/common/src/leap/soledad/common/tests/test_soledad.py @@ -100,6 +100,7 @@ class AuxMethodsTestCase(BaseSoledadTest): self.assertEqual('value_1', sol._server_url) sol.close() + @defer.inlineCallbacks def test_change_passphrase(self): """ Test if passphrase can be changed. @@ -111,38 +112,25 @@ class AuxMethodsTestCase(BaseSoledadTest): prefix=prefix, ) - def _change_passphrase(doc1): - self._doc1 = doc1 - sol.change_passphrase(u'654321') - sol.close() - - def _assert_wrong_password_raises(results): - with self.assertRaises(DatabaseAccessError): - self._soledad_instance( - 'leap@leap.se', - passphrase=u'123', - prefix=prefix) + doc1 = yield sol.create_doc({'simple': 'doc'}) + sol.change_passphrase(u'654321') + sol.close() - def _instantiate_with_new_passphrase(results): - sol2 = self._soledad_instance( + with self.assertRaises(DatabaseAccessError): + self._soledad_instance( 'leap@leap.se', - passphrase=u'654321', + passphrase=u'123', prefix=prefix) - self._sol2 = sol2 - return sol2.get_doc(self._doc1.doc_id) - def _assert_docs_are_equal(doc2): - self.assertEqual(self._doc1, doc2) - self._sol2.close() + sol2 = self._soledad_instance( + 'leap@leap.se', + passphrase=u'654321', + prefix=prefix) + doc2 = yield sol2.get_doc(doc1.doc_id) - d = sol.create_doc({'simple': 'doc'}) - d.addCallback(_change_passphrase) - d.addCallback(_assert_wrong_password_raises) - d.addCallback(_instantiate_with_new_passphrase) - d.addCallback(_assert_docs_are_equal) - d.addCallback(lambda _: sol.close()) + self.assertEqual(doc1, doc2) - return d + sol2.close() def test_change_passphrase_with_short_passphrase_raises(self): """ -- cgit v1.2.3 From 3f9e88b0c4d03aba4e406143813777f14635e20d Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 12 May 2016 10:24:42 -0400 Subject: [docs] add note about server token format --- docs/server-token.txt | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 docs/server-token.txt diff --git a/docs/server-token.txt b/docs/server-token.txt new file mode 100644 index 00000000..89e4d69f --- /dev/null +++ b/docs/server-token.txt @@ -0,0 +1,8 @@ +Requests to the soledad server use a slightly different format than bonafide: + +
+Authentication: 'Token <[base64-encoded]uid:token>'
+
+ +where @<...>@ is a base64-encoded string that concatenates the user id and the +token. -- cgit v1.2.3 From 3e4870f8bd6186f3d0821f392a0dafc5d5247ad4 Mon Sep 17 00:00:00 2001 From: Tulio Casagrande Date: Tue, 31 May 2016 19:55:43 -0300 Subject: [style] remove misused lambdas Pep8 was warning about assignment of lambdas. These lambdas should be partials --- common/src/leap/soledad/common/backend.py | 5 ++--- common/src/leap/soledad/common/couch/__init__.py | 3 ++- common/src/leap/soledad/common/tests/test_command.py | 5 ++++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/common/src/leap/soledad/common/backend.py b/common/src/leap/soledad/common/backend.py index 53426fb5..0a36c068 100644 --- a/common/src/leap/soledad/common/backend.py +++ b/common/src/leap/soledad/common/backend.py @@ -18,7 +18,7 @@ """A U1DB generic backend.""" - +import functools from u1db import vectorclock from u1db.errors import ( RevisionConflict, @@ -438,9 +438,8 @@ class SoledadBackend(CommonBackend): generation. :type other_transaction_id: str """ - function = self._set_replica_gen_and_trans_id args = [other_replica_uid, other_generation, other_transaction_id] - callback = lambda: function(*args) + callback = functools.partial(self._set_replica_gen_and_trans_id, *args) if self.batching: self.after_batch_callbacks['set_source_info'] = callback else: diff --git a/common/src/leap/soledad/common/couch/__init__.py b/common/src/leap/soledad/common/couch/__init__.py index 18ed8a19..5bda8071 100644 --- a/common/src/leap/soledad/common/couch/__init__.py +++ b/common/src/leap/soledad/common/couch/__init__.py @@ -24,6 +24,7 @@ import re import uuid import binascii import time +import functools from StringIO import StringIO @@ -340,7 +341,7 @@ class CouchDatabase(object): # This will not be needed when/if we switch from python-couchdb to # paisley. time.strptime('Mar 8 1917', '%b %d %Y') - get_one = lambda doc_id: self.get_doc(doc_id, check_for_conflicts) + get_one = functools.partial(self.get_doc, check_for_conflicts=check_for_conflicts) docs = [THREAD_POOL.apply_async(get_one, [doc_id]) for doc_id in doc_ids] for doc in docs: diff --git a/common/src/leap/soledad/common/tests/test_command.py b/common/src/leap/soledad/common/tests/test_command.py index c386bdd2..2136bb8f 100644 --- a/common/src/leap/soledad/common/tests/test_command.py +++ b/common/src/leap/soledad/common/tests/test_command.py @@ -21,10 +21,13 @@ from twisted.trial import unittest from leap.soledad.common.command import exec_validated_cmd +def validator(arg): + return True if arg is 'valid' else False + + class ExecuteValidatedCommandTest(unittest.TestCase): def test_argument_validation(self): - validator = lambda arg: True if arg is 'valid' else False status, out = exec_validated_cmd("command", "invalid arg", validator) self.assertEquals(status, 1) self.assertEquals(out, "invalid argument") -- cgit v1.2.3 From f829832457237b7342e510e4112f66819be3ab3d Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 30 May 2016 13:45:51 -0300 Subject: [test] add files to create docker image --- scripts/docker/Dockerfile | 32 ++ scripts/docker/Makefile | 30 + scripts/docker/README.md | 30 + scripts/docker/TODO | 3 + scripts/docker/files/conf/cert_default.conf | 15 + scripts/docker/files/conf/couchdb_default.ini | 361 ++++++++++++ .../docker/files/conf/soledad-server_default.conf | 5 + scripts/docker/files/leap.list | 4 + scripts/docker/files/setup-env.sh | 44 ++ scripts/docker/files/start-server.sh | 25 + scripts/docker/files/test-env.py | 639 +++++++++++++++++++++ scripts/docker/helper/get-container-ip.sh | 18 + 12 files changed, 1206 insertions(+) create mode 100644 scripts/docker/Dockerfile create mode 100644 scripts/docker/Makefile create mode 100644 scripts/docker/README.md create mode 100644 scripts/docker/TODO create mode 100644 scripts/docker/files/conf/cert_default.conf create mode 100644 scripts/docker/files/conf/couchdb_default.ini create mode 100644 scripts/docker/files/conf/soledad-server_default.conf create mode 100644 scripts/docker/files/leap.list create mode 100755 scripts/docker/files/setup-env.sh create mode 100755 scripts/docker/files/start-server.sh create mode 100755 scripts/docker/files/test-env.py create mode 100755 scripts/docker/helper/get-container-ip.sh diff --git a/scripts/docker/Dockerfile b/scripts/docker/Dockerfile new file mode 100644 index 00000000..8d462db9 --- /dev/null +++ b/scripts/docker/Dockerfile @@ -0,0 +1,32 @@ +# start with a fresh debian image +FROM debian + +# expose soledad server port in case we want to run a server container +EXPOSE 2424 + +# install dependencies from debian repos +COPY files/leap.list /etc/apt/sources.list.d/ + +RUN apt-get update +RUN apt-get -y --force-yes install leap-archive-keyring + +RUN apt-get update +RUN apt-get -y install git +RUN apt-get -y install libpython2.7-dev +RUN apt-get -y install libffi-dev +RUN apt-get -y install libssl-dev +RUN apt-get -y install libzmq3-dev +RUN apt-get -y install python-pip +RUN apt-get -y install couchdb + +# copy over files to help setup the environment and run soledad +RUN mkdir -p /usr/local/soledad +RUN mkdir -p /usr/local/soledad/conf + +COPY files/setup-env.sh /usr/local/soledad/ +COPY files/test-env.py /usr/local/soledad/ +COPY files/start-server.sh /usr/local/soledad/ +COPY files/conf/* /usr/local/soledad/conf/ + +# clone repos and install dependencies from leap wheels using pip +RUN /usr/local/soledad/setup-env.sh diff --git a/scripts/docker/Makefile b/scripts/docker/Makefile new file mode 100644 index 00000000..7a80fe02 --- /dev/null +++ b/scripts/docker/Makefile @@ -0,0 +1,30 @@ +#/usr/bin/env + +IMAGE_NAME = "leap/soledad:1.0" + +all: image + +image: + docker build -t $(IMAGE_NAME) . + +run-server: image + rm -f $(CONTAINER_ID_FILE) + docker run \ + --env="SOLEDAD_REMOTE=https://0xacab.org/leap/soledad.git" \ + --env="SOLEDAD_BRANCH=develop" \ + --cidfile=$(CONTAINER_ID_FILE) \ + --detach \ + $(IMAGE_NAME) \ + /usr/local/soledad/start-server.sh + +# TODO: the following rule does not work for now, we have to add a +# `start-test.sh` file +run-test: image + container_id=`cat $(CONTAINER_ID_FILE)`; \ + server_ip=`./helper/get-container-ip.sh $${container_id}`; \ + docker run \ + --env="SOLEDAD_REMOTE=https://0xacab.org/leap/soledad.git" \ + --env="SOLEDAD_BRANCH=develop" \ + --env="SOLEDAD_SERVER_IP=$${server_ip}" \ + $(IMAGE_NAME) \ + /usr/local/soledad/start-test.sh diff --git a/scripts/docker/README.md b/scripts/docker/README.md new file mode 100644 index 00000000..d15129fa --- /dev/null +++ b/scripts/docker/README.md @@ -0,0 +1,30 @@ +Soledad Docker Images +===================== + +The files in this directory help create a docker image that is usable for +running soledad server and client in an isolated docker context. This is +especially useful for testing purposes as you can limit/reserve a certain +amount of resources for the soledad process, and thus provide a baseline for +comparison of time and resource consumption between distinct runs. + +Check the `Dockerfile` for the rules for building the docker image. + +Check the `Makefile` for example usage of the files in this directory. + + +Environment variables for server script +--------------------------------------- + +If you want to run the image for testing you may pass the following +environment variables for the `files/start-server.sh` script for checking out +a specific branch on the soledad repository: + + SOLEDAD_REMOTE - a git url for a remote repository that is added at run time + to the local soledad git repository. + + SOLEDAD_BRANCH - the name of a branch to be checked out from the configured + remote repository. + +Example: + + docker run leap/soledad:1.0 /usr/local/soledad/start-server.sh diff --git a/scripts/docker/TODO b/scripts/docker/TODO new file mode 100644 index 00000000..75d45a8e --- /dev/null +++ b/scripts/docker/TODO @@ -0,0 +1,3 @@ +- limit resources of containers (mem and cpu) +- add a file to run tests inside container +- use server ip to run test diff --git a/scripts/docker/files/conf/cert_default.conf b/scripts/docker/files/conf/cert_default.conf new file mode 100644 index 00000000..8043cea3 --- /dev/null +++ b/scripts/docker/files/conf/cert_default.conf @@ -0,0 +1,15 @@ +[ req ] +default_bits = 1024 +default_keyfile = keyfile.pem +distinguished_name = req_distinguished_name +prompt = no +output_password = mypass + +[ req_distinguished_name ] +C = GB +ST = Test State or Province +L = Test Locality +O = Organization Name +OU = Organizational Unit Name +CN = localhost +emailAddress = test@email.address diff --git a/scripts/docker/files/conf/couchdb_default.ini b/scripts/docker/files/conf/couchdb_default.ini new file mode 100644 index 00000000..5ab72d7b --- /dev/null +++ b/scripts/docker/files/conf/couchdb_default.ini @@ -0,0 +1,361 @@ +; etc/couchdb/default.ini.tpl. Generated from default.ini.tpl.in by configure. + +; Upgrading CouchDB will overwrite this file. +[vendor] +name = The Apache Software Foundation +version = 1.6.0 + +[couchdb] +database_dir = BASEDIR +view_index_dir = BASEDIR +util_driver_dir = /usr/lib/x86_64-linux-gnu/couchdb/erlang/lib/couch-1.6.0/priv/lib +max_document_size = 4294967296 ; 4 GB +os_process_timeout = 5000 ; 5 seconds. for view and external servers. +max_dbs_open = 100 +delayed_commits = true ; set this to false to ensure an fsync before 201 Created is returned +uri_file = BASEDIR/couch.uri +; Method used to compress everything that is appended to database and view index files, except +; for attachments (see the attachments section). Available methods are: +; +; none - no compression +; snappy - use google snappy, a very fast compressor/decompressor +uuid = bc2f8b84ecb0b13a31cf7f6881a52194 + +; deflate_[N] - use zlib's deflate, N is the compression level which ranges from 1 (fastest, +; lowest compression ratio) to 9 (slowest, highest compression ratio) +file_compression = snappy +; Higher values may give better read performance due to less read operations +; and/or more OS page cache hits, but they can also increase overall response +; time for writes when there are many attachment write requests in parallel. +attachment_stream_buffer_size = 4096 + +plugin_dir = /usr/lib/x86_64-linux-gnu/couchdb/plugins + +[database_compaction] +; larger buffer sizes can originate smaller files +doc_buffer_size = 524288 ; value in bytes +checkpoint_after = 5242880 ; checkpoint after every N bytes were written + +[view_compaction] +; larger buffer sizes can originate smaller files +keyvalue_buffer_size = 2097152 ; value in bytes + +[httpd] +port = 5984 +bind_address = 127.0.0.1 +authentication_handlers = {couch_httpd_oauth, oauth_authentication_handler}, {couch_httpd_auth, cookie_authentication_handler}, {couch_httpd_auth, default_authentication_handler} +default_handler = {couch_httpd_db, handle_request} +secure_rewrites = true +vhost_global_handlers = _utils, _uuids, _session, _oauth, _users +allow_jsonp = false +; Options for the MochiWeb HTTP server. +;server_options = [{backlog, 128}, {acceptor_pool_size, 16}] +; For more socket options, consult Erlang's module 'inet' man page. +;socket_options = [{recbuf, 262144}, {sndbuf, 262144}, {nodelay, true}] +socket_options = [{recbuf, 262144}, {sndbuf, 262144}] +log_max_chunk_size = 1000000 +enable_cors = false +; CouchDB can optionally enforce a maximum uri length; +; max_uri_length = 8000 + +[ssl] +port = 6984 + +[log] +file = BASEDIR/couch.log +level = info +include_sasl = true + +[couch_httpd_auth] +authentication_db = _users +authentication_redirect = /_utils/session.html +require_valid_user = false +timeout = 600 ; number of seconds before automatic logout +auth_cache_size = 50 ; size is number of cache entries +allow_persistent_cookies = false ; set to true to allow persistent cookies +iterations = 10 ; iterations for password hashing +; min_iterations = 1 +; max_iterations = 1000000000 +; comma-separated list of public fields, 404 if empty +; public_fields = + +[cors] +credentials = false +; List of origins separated by a comma, * means accept all +; Origins must include the scheme: http://example.com +; You can’t set origins: * and credentials = true at the same time. +;origins = * +; List of accepted headers separated by a comma +; headers = +; List of accepted methods +; methods = + + +; Configuration for a vhost +;[cors:http://example.com] +; credentials = false +; List of origins separated by a comma +; Origins must include the scheme: http://example.com +; You can’t set origins: * and credentials = true at the same time. +;origins = +; List of accepted headers separated by a comma +; headers = +; List of accepted methods +; methods = + +[couch_httpd_oauth] +; If set to 'true', oauth token and consumer secrets will be looked up +; in the authentication database (_users). These secrets are stored in +; a top level property named "oauth" in user documents. Example: +; { +; "_id": "org.couchdb.user:joe", +; "type": "user", +; "name": "joe", +; "password_sha": "fe95df1ca59a9b567bdca5cbaf8412abd6e06121", +; "salt": "4e170ffeb6f34daecfd814dfb4001a73" +; "roles": ["foo", "bar"], +; "oauth": { +; "consumer_keys": { +; "consumerKey1": "key1Secret", +; "consumerKey2": "key2Secret" +; }, +; "tokens": { +; "token1": "token1Secret", +; "token2": "token2Secret" +; } +; } +; } +use_users_db = false + +[query_servers] +javascript = /usr/bin/couchjs /usr/share/couchdb/server/main.js +coffeescript = /usr/bin/couchjs /usr/share/couchdb/server/main-coffee.js + + +; Changing reduce_limit to false will disable reduce_limit. +; If you think you're hitting reduce_limit with a "good" reduce function, +; please let us know on the mailing list so we can fine tune the heuristic. +[query_server_config] +reduce_limit = true +os_process_limit = 25 + +[daemons] +index_server={couch_index_server, start_link, []} +external_manager={couch_external_manager, start_link, []} +query_servers={couch_query_servers, start_link, []} +vhosts={couch_httpd_vhost, start_link, []} +httpd={couch_httpd, start_link, []} +stats_aggregator={couch_stats_aggregator, start, []} +stats_collector={couch_stats_collector, start, []} +uuids={couch_uuids, start, []} +auth_cache={couch_auth_cache, start_link, []} +replicator_manager={couch_replicator_manager, start_link, []} +os_daemons={couch_os_daemons, start_link, []} +compaction_daemon={couch_compaction_daemon, start_link, []} + +[httpd_global_handlers] +/ = {couch_httpd_misc_handlers, handle_welcome_req, <<"Welcome">>} +favicon.ico = {couch_httpd_misc_handlers, handle_favicon_req, "/usr/share/couchdb/www"} + +_utils = {couch_httpd_misc_handlers, handle_utils_dir_req, "/usr/share/couchdb/www"} +_all_dbs = {couch_httpd_misc_handlers, handle_all_dbs_req} +_active_tasks = {couch_httpd_misc_handlers, handle_task_status_req} +_config = {couch_httpd_misc_handlers, handle_config_req} +_replicate = {couch_replicator_httpd, handle_req} +_uuids = {couch_httpd_misc_handlers, handle_uuids_req} +_restart = {couch_httpd_misc_handlers, handle_restart_req} +_stats = {couch_httpd_stats_handlers, handle_stats_req} +_log = {couch_httpd_misc_handlers, handle_log_req} +_session = {couch_httpd_auth, handle_session_req} +_oauth = {couch_httpd_oauth, handle_oauth_req} +_db_updates = {couch_dbupdates_httpd, handle_req} +_plugins = {couch_plugins_httpd, handle_req} + +[httpd_db_handlers] +_all_docs = {couch_mrview_http, handle_all_docs_req} +_changes = {couch_httpd_db, handle_changes_req} +_compact = {couch_httpd_db, handle_compact_req} +_design = {couch_httpd_db, handle_design_req} +_temp_view = {couch_mrview_http, handle_temp_view_req} +_view_cleanup = {couch_mrview_http, handle_cleanup_req} + +; The external module takes an optional argument allowing you to narrow it to a +; single script. Otherwise the script name is inferred from the first path section +; after _external's own path. +; _mypath = {couch_httpd_external, handle_external_req, <<"mykey">>} +; _external = {couch_httpd_external, handle_external_req} + +[httpd_design_handlers] +_compact = {couch_mrview_http, handle_compact_req} +_info = {couch_mrview_http, handle_info_req} +_list = {couch_mrview_show, handle_view_list_req} +_rewrite = {couch_httpd_rewrite, handle_rewrite_req} +_show = {couch_mrview_show, handle_doc_show_req} +_update = {couch_mrview_show, handle_doc_update_req} +_view = {couch_mrview_http, handle_view_req} + +; enable external as an httpd handler, then link it with commands here. +; note, this api is still under consideration. +; [external] +; mykey = /path/to/mycommand + +; Here you can setup commands for CouchDB to manage +; while it is alive. It will attempt to keep each command +; alive if it exits. +; [os_daemons] +; some_daemon_name = /path/to/script -with args + + +[uuids] +; Known algorithms: +; random - 128 bits of random awesome +; All awesome, all the time. +; sequential - monotonically increasing ids with random increments +; First 26 hex characters are random. Last 6 increment in +; random amounts until an overflow occurs. On overflow, the +; random prefix is regenerated and the process starts over. +; utc_random - Time since Jan 1, 1970 UTC with microseconds +; First 14 characters are the time in hex. Last 18 are random. +; utc_id - Time since Jan 1, 1970 UTC with microseconds, plus utc_id_suffix string +; First 14 characters are the time in hex. uuids/utc_id_suffix string value is appended to these. +algorithm = sequential +; The utc_id_suffix value will be appended to uuids generated by the utc_id algorithm. +; Replicating instances should have unique utc_id_suffix values to ensure uniqueness of utc_id ids. +utc_id_suffix = +# Maximum number of UUIDs retrievable from /_uuids in a single request +max_count = 1000 + +[stats] +; rate is in milliseconds +rate = 1000 +; sample intervals are in seconds +samples = [0, 60, 300, 900] + +[attachments] +compression_level = 8 ; from 1 (lowest, fastest) to 9 (highest, slowest), 0 to disable compression +compressible_types = text/*, application/javascript, application/json, application/xml + +[replicator] +db = _replicator +; Maximum replicaton retry count can be a non-negative integer or "infinity". +max_replication_retry_count = 10 +; More worker processes can give higher network throughput but can also +; imply more disk and network IO. +worker_processes = 4 +; With lower batch sizes checkpoints are done more frequently. Lower batch sizes +; also reduce the total amount of used RAM memory. +worker_batch_size = 500 +; Maximum number of HTTP connections per replication. +http_connections = 20 +; HTTP connection timeout per replication. +; Even for very fast/reliable networks it might need to be increased if a remote +; database is too busy. +connection_timeout = 30000 +; If a request fails, the replicator will retry it up to N times. +retries_per_request = 10 +; Some socket options that might boost performance in some scenarios: +; {nodelay, boolean()} +; {sndbuf, integer()} +; {recbuf, integer()} +; {priority, integer()} +; See the `inet` Erlang module's man page for the full list of options. +socket_options = [{keepalive, true}, {nodelay, false}] +; Path to a file containing the user's certificate. +;cert_file = /full/path/to/server_cert.pem +; Path to file containing user's private PEM encoded key. +;key_file = /full/path/to/server_key.pem +; String containing the user's password. Only used if the private keyfile is password protected. +;password = somepassword +; Set to true to validate peer certificates. +verify_ssl_certificates = false +; File containing a list of peer trusted certificates (in the PEM format). +;ssl_trusted_certificates_file = /etc/ssl/certs/ca-certificates.crt +; Maximum peer certificate depth (must be set even if certificate validation is off). +ssl_certificate_max_depth = 3 + +[compaction_daemon] +; The delay, in seconds, between each check for which database and view indexes +; need to be compacted. +check_interval = 300 +; If a database or view index file is smaller then this value (in bytes), +; compaction will not happen. Very small files always have a very high +; fragmentation therefore it's not worth to compact them. +min_file_size = 131072 + +[compactions] +; List of compaction rules for the compaction daemon. +; The daemon compacts databases and their respective view groups when all the +; condition parameters are satisfied. Configuration can be per database or +; global, and it has the following format: +; +; database_name = [ {ParamName, ParamValue}, {ParamName, ParamValue}, ... ] +; _default = [ {ParamName, ParamValue}, {ParamName, ParamValue}, ... ] +; +; Possible parameters: +; +; * db_fragmentation - If the ratio (as an integer percentage), of the amount +; of old data (and its supporting metadata) over the database +; file size is equal to or greater then this value, this +; database compaction condition is satisfied. +; This value is computed as: +; +; (file_size - data_size) / file_size * 100 +; +; The data_size and file_size values can be obtained when +; querying a database's information URI (GET /dbname/). +; +; * view_fragmentation - If the ratio (as an integer percentage), of the amount +; of old data (and its supporting metadata) over the view +; index (view group) file size is equal to or greater then +; this value, then this view index compaction condition is +; satisfied. This value is computed as: +; +; (file_size - data_size) / file_size * 100 +; +; The data_size and file_size values can be obtained when +; querying a view group's information URI +; (GET /dbname/_design/groupname/_info). +; +; * from _and_ to - The period for which a database (and its view groups) compaction +; is allowed. The value for these parameters must obey the format: +; +; HH:MM - HH:MM (HH in [0..23], MM in [0..59]) +; +; * strict_window - If a compaction is still running after the end of the allowed +; period, it will be canceled if this parameter is set to 'true'. +; It defaults to 'false' and it's meaningful only if the *period* +; parameter is also specified. +; +; * parallel_view_compaction - If set to 'true', the database and its views are +; compacted in parallel. This is only useful on +; certain setups, like for example when the database +; and view index directories point to different +; disks. It defaults to 'false'. +; +; Before a compaction is triggered, an estimation of how much free disk space is +; needed is computed. This estimation corresponds to 2 times the data size of +; the database or view index. When there's not enough free disk space to compact +; a particular database or view index, a warning message is logged. +; +; Examples: +; +; 1) [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}] +; The `foo` database is compacted if its fragmentation is 70% or more. +; Any view index of this database is compacted only if its fragmentation +; is 60% or more. +; +; 2) [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "00:00"}, {to, "04:00"}] +; Similar to the preceding example but a compaction (database or view index) +; is only triggered if the current time is between midnight and 4 AM. +; +; 3) [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "00:00"}, {to, "04:00"}, {strict_window, true}] +; Similar to the preceding example - a compaction (database or view index) +; is only triggered if the current time is between midnight and 4 AM. If at +; 4 AM the database or one of its views is still compacting, the compaction +; process will be canceled. +; +; 4) [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "00:00"}, {to, "04:00"}, {strict_window, true}, {parallel_view_compaction, true}] +; Similar to the preceding example, but a database and its views can be +; compacted in parallel. +; +;_default = [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "23:00"}, {to, "04:00"}] diff --git a/scripts/docker/files/conf/soledad-server_default.conf b/scripts/docker/files/conf/soledad-server_default.conf new file mode 100644 index 00000000..5e286374 --- /dev/null +++ b/scripts/docker/files/conf/soledad-server_default.conf @@ -0,0 +1,5 @@ +[soledad-server] +couch_url = http://localhost:5984 +create_cmd = sudo -u soledad-admin /usr/bin/create-user-db +admin_netrc = /etc/couchdb/couchdb-soledad-admin.netrc +batching = 0 diff --git a/scripts/docker/files/leap.list b/scripts/docker/files/leap.list new file mode 100644 index 00000000..7eb474d8 --- /dev/null +++ b/scripts/docker/files/leap.list @@ -0,0 +1,4 @@ +# This file is meant to be copied into the `/etc/apt/sources.list.d` directory +# inside a docker image to provide a source for leap-specific packages. + +deb http://deb.leap.se/0.8 jessie main diff --git a/scripts/docker/files/setup-env.sh b/scripts/docker/files/setup-env.sh new file mode 100755 index 00000000..c98a6d08 --- /dev/null +++ b/scripts/docker/files/setup-env.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Clone soledad repository and install soledad dependencies needed to run +# client and server in a test environment. +# +# This script is meant to be copied to the docker container and run after +# system dependencies have been installed. + +BASEDIR="/var/local" +BASEURL="https://github.com/leapcode" + +mkdir -p ${BASEDIR} + +# clone repositories +repos="soledad leap_pycommon" + +for repo in ${repos}; do + repodir=${BASEDIR}/${repo} + if [ ! -d ${repodir} ]; then + git clone ${BASEURL}/${repo} ${repodir} + git -C ${repodir} fetch origin + fi +done + +# use latest pip because the version available in debian jessie doesn't +# support wheels +pip install -U pip + +pip install psutil + +# install dependencies and packages +install_script="pkg/pip_install_requirements.sh" +opts="--use-leap-wheels" +pkgs="leap_pycommon soledad/common soledad/client soledad/server" + +for pkg in ${pkgs}; do + pkgdir=${BASEDIR}/${pkg} + testing="" + if [ -f ${pkgdir}/pkg/requirements-testing.pip ]; then + testing="--testing" + fi + (cd ${pkgdir} && ${install_script} ${testing} ${opts}) + (cd ${pkgdir} && python setup.py develop) +done diff --git a/scripts/docker/files/start-server.sh b/scripts/docker/files/start-server.sh new file mode 100755 index 00000000..ea14aa5a --- /dev/null +++ b/scripts/docker/files/start-server.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +# Start a soledad server inside a docker container. +# +# This script is meant to be copied to the docker container and run upon +# container start. + +CMD="/usr/local/soledad/test-env.py" +REPO="/var/local/soledad" + +if [ ! -z "${SOLEDAD_REMOTE}" ]; then + git -C ${REPO} remote add test ${SOLEDAD_REMOTE} + git -C ${REPO} fetch test +fi + +if [ ! -z "${SOLEDAD_BRANCH}" ]; then + git -C ${REPO} checkout ${SOLEDAD_BRANCH} +fi + +${CMD} couch start +${CMD} user-db create +${CMD} token-db create +${CMD} token-db insert-token +${CMD} cert create +${CMD} soledad-server start --no-daemonize diff --git a/scripts/docker/files/test-env.py b/scripts/docker/files/test-env.py new file mode 100755 index 00000000..6ff0a4ba --- /dev/null +++ b/scripts/docker/files/test-env.py @@ -0,0 +1,639 @@ +#!/usr/bin/env python + + +""" +This script knows how to build a minimum environment for Soledad Server, which +includes the following: + + - Couch server startup + - Token and shared database initialization + - Soledad Server startup + +Options can be passed for configuring the different environments, so this may +be used by other programs to setup different environments for arbitrary tests. +Use the --help option to get information on usage. + +For some commands you will need an environment with Soledad python packages +available, thus you might want to explicitly call python and not rely in the +shebang line. +""" + + +import time +import os +import signal +import tempfile +import psutil +from argparse import ArgumentParser +from subprocess import call +from couchdb import Server +from couchdb.http import PreconditionFailed +from couchdb.http import ResourceConflict +from couchdb.http import ResourceNotFound +from hashlib import sha512 +from u1db.errors import DatabaseDoesNotExist + + +# +# Utilities +# + +def get_pid(pidfile): + if not os.path.isfile(pidfile): + return 0 + try: + with open(pidfile) as f: + return int(f.read()) + except IOError: + return 0 + + +def pid_is_running(pid): + try: + psutil.Process(pid) + return True + except psutil.NoSuchProcess: + return False + + +def pidfile_is_running(pidfile): + try: + pid = get_pid(pidfile) + psutil.Process(pid) + return pid + except psutil.NoSuchProcess: + return False + + +def status_from_pidfile(args, default_basedir): + basedir = _get_basedir(args, default_basedir) + pidfile = os.path.join(basedir, args.pidfile) + try: + pid = get_pid(pidfile) + psutil.Process(pid) + print "[+] running - pid: %d" % pid + except (IOError, psutil.NoSuchProcess): + print "[-] stopped" + + +def kill_all_executables(args): + basename = os.path.basename(args.executable) + pids = [int(pid) for pid in os.listdir('/proc') if pid.isdigit()] + for pid in pids: + try: + p = psutil.Process(pid) + if p.name() == basename: + print '[!] killing - pid: %d' % pid + os.kill(pid, signal.SIGKILL) + except: + pass + + +# +# Couch Server control +# + +COUCH_EXECUTABLE = '/usr/bin/couchdb' +ERLANG_EXECUTABLE = 'beam.smp' +COUCH_TEMPLATE = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + './conf/couchdb_default.ini') +COUCH_TEMPLATE +COUCH_PIDFILE = 'couchdb.pid' +COUCH_LOGFILE = 'couchdb.log' +COUCH_PORT = 5984 +COUCH_HOST = '127.0.0.1' +COUCH_BASEDIR = '/tmp/couch_test' + + +def _get_basedir(args, default): + basedir = args.basedir + if not basedir: + basedir = default + if not os.path.isdir(basedir): + os.mkdir(basedir) + return basedir + + +def couch_server_start(args): + basedir = _get_basedir(args, COUCH_BASEDIR) + pidfile = os.path.join(basedir, args.pidfile) + logfile = os.path.join(basedir, args.logfile) + + # check if already running + pid = get_pid(pidfile) + if pid_is_running(pid): + print '[*] error: already running - pid: %d' % pid + exit(1) + if os.path.isfile(pidfile): + os.unlink(pidfile) + + # generate a config file from template if needed + config_file = args.config_file + if not config_file: + config_file = tempfile.mktemp(prefix='couch_config_', dir=basedir) + lines = [] + with open(args.template) as f: + lines = f.readlines() + lines = map(lambda l: l.replace('BASEDIR', basedir), lines) + with open(config_file, 'w') as f: + f.writelines(lines) + + # start couch server + try: + call([ + args.executable, + '-n', # reset configuration file chain (including system default) + '-a %s' % config_file, # add configuration FILE to chain + '-b', # spawn as a background process + '-p %s' % pidfile, # set the background PID FILE + '-o %s' % logfile, # redirect background stdout to FILE + '-e %s' % logfile]) # redirect background stderr to FILE + except Exception as e: + print '[*] error: could not start couch server - %s' % str(e) + exit(1) + + # couch may take a bit to store the pid in the pidfile, so we just wait + # until it does + pid = None + while not pid: + try: + pid = get_pid(pidfile) + break + except: + time.sleep(0.1) + + print '[+] running - pid: %d' % pid + + +def couch_server_stop(args): + basedir = _get_basedir(args, COUCH_BASEDIR) + pidfile = os.path.join(basedir, args.pidfile) + pid = get_pid(pidfile) + if not pid_is_running(pid): + print '[*] error: no running server found' + exit(1) + call([ + args.executable, + '-p %s' % pidfile, # set the background PID FILE + '-k']) # kill the background process, will respawn if needed + print '[-] stopped - pid: %d ' % pid + + +def couch_status_from_pidfile(args): + status_from_pidfile(args, COUCH_BASEDIR) + + +# +# User DB maintenance # +# + +def user_db_create(args): + from leap.soledad.common.couch import CouchDatabase + url = 'http://localhost:%d/user-%s' % (args.port, args.uuid) + try: + CouchDatabase.open_database( + url=url, create=False, replica_uid=None, ensure_ddocs=True) + print '[*] error: database "user-%s" already exists' % args.uuid + exit(1) + except DatabaseDoesNotExist: + CouchDatabase.open_database( + url=url, create=True, replica_uid=None, ensure_ddocs=True) + print '[+] database created: user-%s' % args.uuid + + +def user_db_delete(args): + s = _couch_get_server(args) + try: + dbname = 'user-%s' % args.uuid + s.delete(dbname) + print '[-] database deleted: %s' % dbname + except ResourceNotFound: + print '[*] error: database "%s" does not exist' % dbname + exit(1) + + +# +# Soledad Server control +# + +TWISTD_EXECUTABLE = 'twistd' # use whatever is available on path + +SOLEDAD_SERVER_BASEDIR = '/tmp/soledad_server_test' +SOLEDAD_SERVER_CONFIG_FILE = './conf/soledad_default.ini' +SOLEDAD_SERVER_PIDFILE = 'soledad.pid' +SOLEDAD_SERVER_LOGFILE = 'soledad.log' +SOLEDAD_SERVER_PRIVKEY = 'soledad_privkey.pem' +SOLEDAD_SERVER_CERTKEY = 'soledad_certkey.pem' +SOLEDAD_SERVER_PORT = 2424 +SOLEDAD_SERVER_AUTH_TOKEN = 'an-auth-token' +SOLEDAD_SERVER_URL = 'https://localhost:2424' + +SOLEDAD_CLIENT_PASS = '12345678' +SOLEDAD_CLIENT_BASEDIR = '/tmp/soledad_client_test' +SOLEDAD_CLIENT_UUID = '1234567890abcdef' + + +def soledad_server_start(args): + basedir = _get_basedir(args, SOLEDAD_SERVER_BASEDIR) + pidfile = os.path.join(basedir, args.pidfile) + logfile = os.path.join(basedir, args.logfile) + private_key = os.path.join(basedir, args.private_key) + cert_key = os.path.join(basedir, args.cert_key) + + pid = get_pid(pidfile) + if pid_is_running(pid): + pid = get_pid(pidfile) + print "[*] error: already running - pid: %d" % pid + exit(1) + + port = args.port + if args.tls: + port = 'ssl:%d:privateKey=%s:certKey=%s:sslmethod=SSLv23_METHOD' \ + % (args.port, private_key, cert_key) + params = [ + '--logfile=%s' % logfile, + '--pidfile=%s' % pidfile, + 'web', + '--wsgi=leap.soledad.server.application', + '--port=%s' % port + ] + if args.no_daemonize: + params.insert(0, '--nodaemon') + + call([args.executable] + params) + + pid = get_pid(pidfile) + print '[+] running - pid: %d' % pid + + +def soledad_server_stop(args): + basedir = _get_basedir(args, SOLEDAD_SERVER_BASEDIR) + pidfile = os.path.join(basedir, args.pidfile) + pid = get_pid(pidfile) + if not pid_is_running(pid): + print '[*] error: no running server found' + exit(1) + os.kill(pid, signal.SIGKILL) + print '[-] stopped - pid: %d' % pid + + +def soledad_server_status_from_pidfile(args): + status_from_pidfile(args, SOLEDAD_SERVER_BASEDIR) + + +# couch helpers + +def _couch_get_server(args): + url = 'http://%s:%d/' % (args.host, args.port) + return Server(url=url) + + +def _couch_create_db(args, dbname): + s = _couch_get_server(args) + # maybe create the database + try: + s.create(dbname) + print '[+] database created: %s' % dbname + except PreconditionFailed as e: + error_code, _ = e.message + if error_code == 'file_exists': + print '[*] error: "%s" database already exists' % dbname + exit(1) + return s + + +def _couch_delete_db(args, dbname): + s = _couch_get_server(args) + # maybe create the database + try: + s.delete(dbname) + print '[-] database deleted: %s' % dbname + except ResourceNotFound: + print '[*] error: "%s" database does not exist' % dbname + exit(1) + + +def _token_dbname(): + dbname = 'tokens_' + \ + str(int(time.time() / (30 * 24 * 3600))) + return dbname + + +def token_db_create(args): + dbname = _token_dbname() + _couch_create_db(args, dbname) + + +def token_db_insert_token(args): + s = _couch_get_server(args) + try: + dbname = _token_dbname() + db = s[dbname] + token = sha512(args.auth_token).hexdigest() + db[token] = { + 'type': 'Token', + 'user_id': args.uuid, + } + print '[+] token for uuid "%s" created in tokens database' % args.uuid + except ResourceConflict: + print '[*] error: token for uuid "%s" already exists in tokens database' \ + % args.uuid + exit(1) + + +def token_db_delete(args): + dbname = _token_dbname() + _couch_delete_db(args, dbname) + + +# +# Shared DB creation +# + +def shared_db_create(args): + _couch_create_db(args, 'shared') + + +def shared_db_delete(args): + _couch_delete_db(args, 'shared') + + +# +# Certificate creation +# + +CERT_CONFIG_FILE = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + './conf/cert_default.conf') + + +def cert_create(args): + private_key = os.path.join(args.basedir, args.private_key) + cert_key = os.path.join(args.basedir, args.cert_key) + call([ + 'openssl', + 'req', + '-x509', + '-sha256', + '-nodes', + '-days', '365', + '-newkey', 'rsa:2048', + '-config', args.config_file, + '-keyout', private_key, + '-out', cert_key]) + + +def cert_delete(args): + private_key = os.path.join(args.basedir, args.private_key) + cert_key = os.path.join(args.basedir, args.cert_key) + os.unlink(private_key) + os.unlink(cert_key) + + +# +# Soledad Client Control +# + +def soledad_client_test(args): + + # maybe infer missing parameters + basedir = args.basedir + if not basedir: + basedir = tempfile.mkdtemp() + server_url = args.server_url + if not server_url: + server_url = 'http://127.0.0.1:%d' % args.port + + # get a soledad instance + from client_side_db import _get_soledad_instance + _get_soledad_instance( + args.uuid, + unicode(args.passphrase), + basedir, + server_url, + args.cert_key, + args.auth_token) + + +# +# Command Line Interface +# + +class Command(object): + + def __init__(self, parser=ArgumentParser()): + self.commands = [] + self.parser = parser + self.subparsers = None + + def add_command(self, *args, **kwargs): + # pop out the func parameter to use later + func = None + if 'func' in kwargs.keys(): + func = kwargs.pop('func') + # eventually create a subparser + if not self.subparsers: + self.subparsers = self.parser.add_subparsers() + # create command and associate a function with it + command = Command(self.subparsers.add_parser(*args, **kwargs)) + if func: + command.parser.set_defaults(func=func) + self.commands.append(command) + return command + + def set_func(self, func): + self.parser.set_defaults(func=func) + + def add_argument(self, *args, **kwargs): + self.parser.add_argument(*args, **kwargs) + + def add_arguments(self, arglist): + for args, kwargs in arglist: + self.add_argument(*args, **kwargs) + + def parse_args(self): + return self.parser.parse_args() + + +# +# Command Line Interface +# + +def run_cli(): + cli = Command() + + # couch command with subcommands + cmd_couch = cli.add_command('couch', help="manage couch server") + + cmd_couch_start = cmd_couch.add_command('start', func=couch_server_start) + cmd_couch_start.add_arguments([ + (['--executable', '-e'], {'default': COUCH_EXECUTABLE}), + (['--basedir', '-b'], {}), + (['--config-file', '-c'], {}), + (['--template', '-t'], {'default': COUCH_TEMPLATE}), + (['--pidfile', '-p'], {'default': COUCH_PIDFILE}), + (['--logfile', '-l'], {'default': COUCH_LOGFILE}) + ]) + + cmd_couch_stop = cmd_couch.add_command('stop', func=couch_server_stop) + cmd_couch_stop.add_arguments([ + (['--executable', '-e'], {'default': COUCH_EXECUTABLE}), + (['--basedir', '-b'], {}), + (['--pidfile', '-p'], {'default': COUCH_PIDFILE}), + ]) + + cmd_couch_status = cmd_couch.add_command( + 'status', func=couch_status_from_pidfile) + cmd_couch_status.add_arguments([ + (['--basedir', '-b'], {}), + (['--pidfile', '-p'], {'default': COUCH_PIDFILE})]) + + cmd_couch_kill = cmd_couch.add_command('kill', func=kill_all_executables) + cmd_couch_kill.add_argument( + '--executable', '-e', default=ERLANG_EXECUTABLE) + + # user database maintenance + cmd_user_db = cli.add_command('user-db') + + cmd_user_db_create = cmd_user_db.add_command('create', func=user_db_create) + cmd_user_db_create.add_arguments([ + (['--host', '-H'], {'default': COUCH_HOST}), + (['--port', '-P'], {'type': int, 'default': COUCH_PORT}), + (['--uuid', '-u'], {'default': SOLEDAD_CLIENT_UUID}), + ]) + + cmd_user_db_create = cmd_user_db.add_command( + 'delete', func=user_db_delete) + cmd_user_db_create.add_arguments([ + (['--host', '-H'], {'default': COUCH_HOST}), + (['--port', '-P'], {'type': int, 'default': COUCH_PORT}), + (['--uuid', '-u'], {'default': SOLEDAD_CLIENT_UUID}) + ]) + + # soledad server command with subcommands + cmd_sol_server = cli.add_command( + 'soledad-server', help="manage soledad server") + + cmd_sol_server_start = cmd_sol_server.add_command( + 'start', func=soledad_server_start) + cmd_sol_server_start.add_arguments([ + (['--executable', '-e'], {'default': TWISTD_EXECUTABLE}), + (['--config-file', '-c'], {'default': SOLEDAD_SERVER_CONFIG_FILE}), + (['--pidfile', '-p'], {'default': SOLEDAD_SERVER_PIDFILE}), + (['--logfile', '-l'], {'default': SOLEDAD_SERVER_LOGFILE}), + (['--port', '-P'], {'type': int, 'default': SOLEDAD_SERVER_PORT}), + (['--tls', '-t'], {'action': 'store_true'}), + (['--private-key', '-K'], {'default': SOLEDAD_SERVER_PRIVKEY}), + (['--cert-key', '-C'], {'default': SOLEDAD_SERVER_CERTKEY}), + (['--no-daemonize', '-n'], {'action': 'store_true'}), + (['--basedir', '-b'], {'default': SOLEDAD_SERVER_BASEDIR}), + ]) + + cmd_sol_server_stop = cmd_sol_server.add_command( + 'stop', func=soledad_server_stop) + cmd_sol_server_stop.add_arguments([ + (['--basedir', '-b'], {'default': SOLEDAD_SERVER_BASEDIR}), + (['--pidfile', '-p'], {'default': SOLEDAD_SERVER_PIDFILE}), + ]) + + cmd_sol_server_status = cmd_sol_server.add_command( + 'status', func=soledad_server_status_from_pidfile) + cmd_sol_server_status.add_arguments([ + (['--basedir', '-b'], {'default': SOLEDAD_SERVER_BASEDIR}), + (['--pidfile', '-p'], {'default': SOLEDAD_SERVER_PIDFILE}), + ]) + + cmd_sol_server_kill = cmd_sol_server.add_command( + 'kill', func=kill_all_executables) + cmd_sol_server_kill.add_argument( + '--executable', '-e', default=TWISTD_EXECUTABLE) + + # token db maintenance + cmd_token_db = cli.add_command('token-db') + cmd_token_db_create = cmd_token_db.add_command( + 'create', func=token_db_create) + cmd_token_db_create.add_arguments([ + (['--host', '-H'], {'default': COUCH_HOST}), + (['--uuid', '-u'], {'default': SOLEDAD_CLIENT_UUID}), + (['--port', '-P'], {'type': int, 'default': COUCH_PORT}), + ]) + + cmd_token_db_insert_token = cmd_token_db.add_command( + 'insert-token', func=token_db_insert_token) + cmd_token_db_insert_token.add_arguments([ + (['--host', '-H'], {'default': COUCH_HOST}), + (['--uuid', '-u'], {'default': SOLEDAD_CLIENT_UUID}), + (['--port', '-P'], {'type': int, 'default': COUCH_PORT}), + (['--auth-token', '-a'], {'default': SOLEDAD_SERVER_AUTH_TOKEN}), + ]) + + cmd_token_db_delete = cmd_token_db.add_command( + 'delete', func=token_db_delete) + cmd_token_db_delete.add_arguments([ + (['--host', '-H'], {'default': COUCH_HOST}), + (['--uuid', '-u'], {'default': SOLEDAD_CLIENT_UUID}), + (['--port', '-P'], {'type': int, 'default': COUCH_PORT}), + ]) + + # shared db creation + cmd_shared_db = cli.add_command('shared-db') + + cmd_shared_db_create = cmd_shared_db.add_command( + 'create', func=shared_db_create) + cmd_shared_db_create.add_arguments([ + (['--host', '-H'], {'default': COUCH_HOST}), + (['--port', '-P'], {'type': int, 'default': COUCH_PORT}), + ]) + + cmd_shared_db_delete = cmd_shared_db.add_command( + 'delete', func=shared_db_delete) + cmd_shared_db_delete.add_arguments([ + (['--host', '-H'], {'default': COUCH_HOST}), + (['--port', '-P'], {'type': int, 'default': COUCH_PORT}), + ]) + + # certificate generation + cmd_cert = cli.add_command('cert', help="create tls certificates") + + cmd_cert_create = cmd_cert.add_command('create', func=cert_create) + cmd_cert_create.add_arguments([ + (['--basedir', '-b'], {'default': SOLEDAD_SERVER_BASEDIR}), + (['--config-file', '-c'], {'default': CERT_CONFIG_FILE}), + (['--private-key', '-K'], {'default': SOLEDAD_SERVER_PRIVKEY}), + (['--cert-key', '-C'], {'default': SOLEDAD_SERVER_CERTKEY}), + ]) + + cmd_cert_create = cmd_cert.add_command('delete', func=cert_delete) + cmd_cert_create.add_arguments([ + (['--basedir', '-b'], {'default': SOLEDAD_SERVER_BASEDIR}), + (['--private-key', '-K'], {'default': SOLEDAD_SERVER_PRIVKEY}), + (['--cert-key', '-C'], {'default': SOLEDAD_SERVER_CERTKEY}), + ]) + + # soledad client command with subcommands + cmd_sol_client = cli.add_command( + 'soledad-client', help="manage soledad client") + + cmd_sol_client_test = cmd_sol_client.add_command( + 'test', func=soledad_client_test) + cmd_sol_client_test.add_arguments([ + (['--port', '-P'], {'type': int, 'default': SOLEDAD_SERVER_PORT}), + (['--tls', '-t'], {'action': 'store_true'}), + (['--uuid', '-u'], {'default': SOLEDAD_CLIENT_UUID}), + (['--passphrase', '-k'], {'default': SOLEDAD_CLIENT_PASS}), + (['--basedir', '-b'], {'default': SOLEDAD_CLIENT_BASEDIR}), + (['--server_url', '-s'], {'default': SOLEDAD_SERVER_URL}), + (['--cert-key', '-C'], {'default': os.path.join( + SOLEDAD_SERVER_BASEDIR, + SOLEDAD_SERVER_CERTKEY)}), + (['--auth-token', '-a'], {'default': SOLEDAD_SERVER_AUTH_TOKEN}), + ]) + + # parse and run cli + args = cli.parse_args() + args.func(args) + + +if __name__ == '__main__': + run_cli() diff --git a/scripts/docker/helper/get-container-ip.sh b/scripts/docker/helper/get-container-ip.sh new file mode 100755 index 00000000..2b392350 --- /dev/null +++ b/scripts/docker/helper/get-container-ip.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +# Print the IP of a container to stdout, given its id. Check the output for +# the `docker inspect` commmand for more details: +# +# https://docs.docker.com/engine/reference/commandline/inspect/ + +if [ ${#} -ne 1 ]; then + echo "Usage: ${0} container_id" + exit 1 +fi + +container_id=${1} + +/usr/bin/docker \ + inspect \ + --format='{{.NetworkSettings.IPAddress}}' \ + ${container_id} -- cgit v1.2.3 From 6b34d775613676b2411e1150645a4601c7b3ea46 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 6 Jun 2016 18:06:01 -0300 Subject: [test] add missing deps to Dockerfile --- scripts/docker/Dockerfile | 16 +- scripts/docker/files/client_side_db.py | 322 ++++++++++++++++++++++++++++++ scripts/docker/files/start-client-test.sh | 17 ++ scripts/docker/files/util.py | 75 +++++++ scripts/docker/helper/run-tests.sh | 6 + 5 files changed, 433 insertions(+), 3 deletions(-) create mode 100644 scripts/docker/files/client_side_db.py create mode 100755 scripts/docker/files/start-client-test.sh create mode 100644 scripts/docker/files/util.py create mode 100755 scripts/docker/helper/run-tests.sh diff --git a/scripts/docker/Dockerfile b/scripts/docker/Dockerfile index 8d462db9..24bfff30 100644 --- a/scripts/docker/Dockerfile +++ b/scripts/docker/Dockerfile @@ -18,15 +18,25 @@ RUN apt-get -y install libssl-dev RUN apt-get -y install libzmq3-dev RUN apt-get -y install python-pip RUN apt-get -y install couchdb +RUN apt-get -y install python-srp +RUN apt-get -y install python-scrypt +RUN apt-get -y install leap-keymanager +RUN apt-get -y install python-tz +RUN apt-get -y install curl +RUN apt-get -y install python-ipdb # copy over files to help setup the environment and run soledad RUN mkdir -p /usr/local/soledad RUN mkdir -p /usr/local/soledad/conf +# setup the enviroment for running soledad client and server COPY files/setup-env.sh /usr/local/soledad/ +RUN /usr/local/soledad/setup-env.sh + +# copy runtime files for running server, client, tests, etc on a container COPY files/test-env.py /usr/local/soledad/ +COPY files/client_side_db.py /usr/local/soledad/ +COPY files/util.py /usr/local/soledad/ COPY files/start-server.sh /usr/local/soledad/ +COPY files/start-client-test.sh /usr/local/soledad/ COPY files/conf/* /usr/local/soledad/conf/ - -# clone repos and install dependencies from leap wheels using pip -RUN /usr/local/soledad/setup-env.sh diff --git a/scripts/docker/files/client_side_db.py b/scripts/docker/files/client_side_db.py new file mode 100644 index 00000000..4be33d13 --- /dev/null +++ b/scripts/docker/files/client_side_db.py @@ -0,0 +1,322 @@ +#!/usr/bin/python + +import os +import argparse +import tempfile +import getpass +import requests +import srp._pysrp as srp +import binascii +import logging +import json +import time + +from twisted.internet import reactor +from twisted.internet.defer import inlineCallbacks + +from leap.soledad.client import Soledad +from leap.keymanager import KeyManager +from leap.keymanager.openpgp import OpenPGPKey + +from leap.common.events import server +server.ensure_server() + +from util import ValidateUserHandle + + +""" +Script to give access to client-side Soledad database. + +This is mainly used for tests, but can also be used to recover data from a +Soledad database (public/private keys, export documents, etc). + +To speed up testing/debugging, this script can dump the auth data after +logging in. Use the --export-auth-data option to export auth data to a file. +The contents of the file is a json dictionary containing the uuid, server_url, +cert_file and token, which is enough info to instantiate a soledad client +without having to interact with the webapp again. Use the --use-auth-data +option to use the auth data stored in a file. + +Use the --help option to see available options. +""" + + +# create a logger +logger = logging.getLogger(__name__) +LOG_FORMAT = '%(asctime)s %(message)s' +logging.basicConfig(format=LOG_FORMAT, level=logging.DEBUG) + + +safe_unhexlify = lambda x: binascii.unhexlify(x) if ( + len(x) % 2 == 0) else binascii.unhexlify('0' + x) + + +def _fail(reason): + logger.error('Fail: ' + reason) + exit(2) + + +def _get_api_info(provider): + info = requests.get( + 'https://' + provider + '/provider.json', verify=False).json() + return info['api_uri'], info['api_version'] + + +def _login(username, passphrase, provider, api_uri, api_version): + usr = srp.User(username, passphrase, srp.SHA256, srp.NG_1024) + auth = None + try: + auth = _authenticate(api_uri, api_version, usr).json() + except requests.exceptions.ConnectionError: + _fail('Could not connect to server.') + if 'errors' in auth: + _fail(str(auth['errors'])) + return api_uri, api_version, auth + + +def _authenticate(api_uri, api_version, usr): + api_url = "%s/%s" % (api_uri, api_version) + session = requests.session() + uname, A = usr.start_authentication() + params = {'login': uname, 'A': binascii.hexlify(A)} + init = session.post( + api_url + '/sessions', data=params, verify=False).json() + if 'errors' in init: + _fail('test user not found') + M = usr.process_challenge( + safe_unhexlify(init['salt']), safe_unhexlify(init['B'])) + return session.put(api_url + '/sessions/' + uname, verify=False, + data={'client_auth': binascii.hexlify(M)}) + + +def _get_soledad_info(username, provider, passphrase, basedir): + api_uri, api_version = _get_api_info(provider) + auth = _login(username, passphrase, provider, api_uri, api_version) + # get soledad server url + service_url = '%s/%s/config/soledad-service.json' % \ + (api_uri, api_version) + soledad_hosts = requests.get(service_url, verify=False).json()['hosts'] + hostnames = soledad_hosts.keys() + # allow for choosing the host + host = hostnames[0] + if len(hostnames) > 1: + i = 1 + print "There are many available hosts:" + for h in hostnames: + print " (%d) %s.%s" % (i, h, provider) + i += 1 + choice = raw_input("Choose a host to use (default: 1): ") + if choice != '': + host = hostnames[int(choice) - 1] + server_url = 'https://%s:%d/user-%s' % \ + (soledad_hosts[host]['hostname'], soledad_hosts[host]['port'], + auth[2]['id']) + # get provider ca certificate + ca_cert = requests.get('https://%s/ca.crt' % provider, verify=False).text + cert_file = os.path.join(basedir, 'ca.crt') + with open(cert_file, 'w') as f: + f.write(ca_cert) + return auth[2]['id'], server_url, cert_file, auth[2]['token'] + + +def _get_soledad_instance(uuid, passphrase, basedir, server_url, cert_file, + token): + # setup soledad info + logger.info('UUID is %s' % uuid) + logger.info('Server URL is %s' % server_url) + secrets_path = os.path.join( + basedir, '%s.secret' % uuid) + local_db_path = os.path.join( + basedir, '%s.db' % uuid) + # instantiate soledad + return Soledad( + uuid, + unicode(passphrase), + secrets_path=secrets_path, + local_db_path=local_db_path, + server_url=server_url, + cert_file=cert_file, + auth_token=token, + defer_encryption=True) + + +def _get_keymanager_instance(username, provider, soledad, token, + ca_cert_path=None, api_uri=None, api_version=None, + uid=None, gpgbinary=None): + return KeyManager( + "{username}@{provider}".format(username=username, provider=provider), + "http://uri", + soledad, + token=token, + ca_cert_path=ca_cert_path, + api_uri=api_uri, + api_version=api_version, + uid=uid, + gpgbinary=gpgbinary) + + +def _parse_args(): + # parse command line + parser = argparse.ArgumentParser() + parser.add_argument( + 'user@provider', action=ValidateUserHandle, help='the user handle') + parser.add_argument( + '--basedir', '-b', default=None, + help='soledad base directory') + parser.add_argument( + '--passphrase', '-p', default=None, + help='the user passphrase') + parser.add_argument( + '--get-all-docs', '-a', action='store_true', + help='get all documents from the local database') + parser.add_argument( + '--create-docs', '-c', default=0, type=int, + help='create a number of documents') + parser.add_argument( + '--sync', '-s', action='store_true', + help='synchronize with the server replica') + parser.add_argument( + '--repeat-sync', '-r', action='store_true', + help='repeat synchronization until no new data is received') + parser.add_argument( + '--export-public-key', help="export the public key to a file") + parser.add_argument( + '--export-private-key', help="export the private key to a file") + parser.add_argument( + '--export-incoming-messages', + help="export incoming messages to a directory") + parser.add_argument( + '--export-auth-data', + help="export authentication data to a file") + parser.add_argument( + '--use-auth-data', + help="use authentication data from a file") + return parser.parse_args() + + +def _get_passphrase(args): + passphrase = args.passphrase + if passphrase is None: + passphrase = getpass.getpass( + 'Password for %s@%s: ' % (args.username, args.provider)) + return passphrase + + +def _get_basedir(args): + basedir = args.basedir + if basedir is None: + basedir = tempfile.mkdtemp() + elif not os.path.isdir(basedir): + os.mkdir(basedir) + logger.info('Using %s as base directory.' % basedir) + return basedir + + +@inlineCallbacks +def _export_key(args, km, fname, private=False): + address = args.username + "@" + args.provider + pkey = yield km.get_key( + address, OpenPGPKey, private=private, fetch_remote=False) + with open(args.export_private_key, "w") as f: + f.write(pkey.key_data) + + +@inlineCallbacks +def _export_incoming_messages(soledad, directory): + yield soledad.create_index("by-incoming", "bool(incoming)") + docs = yield soledad.get_from_index("by-incoming", '1') + i = 1 + for doc in docs: + with open(os.path.join(directory, "message_%d.gpg" % i), "w") as f: + f.write(doc.content["_enc_json"]) + i += 1 + + +@inlineCallbacks +def _get_all_docs(soledad): + _, docs = yield soledad.get_all_docs() + for doc in docs: + print json.dumps(doc.content, indent=4) + + +# main program + +@inlineCallbacks +def _main(soledad, km, args): + try: + if args.create_docs: + for i in xrange(args.create_docs): + t = time.time() + logger.debug( + "Creating doc %d/%d..." % (i + 1, args.create_docs)) + content = { + 'datetime': time.strftime( + "%Y-%m-%d %H:%M:%S", time.gmtime(t)), + 'timestamp': t, + 'index': i, + 'total': args.create_docs, + } + yield soledad.create_doc(content) + if args.sync: + yield soledad.sync() + if args.repeat_sync: + old_gen = 0 + new_gen = yield soledad.sync() + while old_gen != new_gen: + old_gen = new_gen + new_gen = yield soledad.sync() + if args.get_all_docs: + yield _get_all_docs(soledad) + if args.export_private_key: + yield _export_key(args, km, args.export_private_key, private=True) + if args.export_public_key: + yield _export_key(args, km, args.expoert_public_key, private=False) + if args.export_incoming_messages: + yield _export_incoming_messages( + soledad, args.export_incoming_messages) + except Exception as e: + logger.error(e) + finally: + soledad.close() + reactor.callWhenRunning(reactor.stop) + + +if __name__ == '__main__': + args = _parse_args() + passphrase = _get_passphrase(args) + basedir = _get_basedir(args) + + if not args.use_auth_data: + # get auth data from server + uuid, server_url, cert_file, token = \ + _get_soledad_info( + args.username, args.provider, passphrase, basedir) + else: + # load auth data from file + with open(args.use_auth_data) as f: + auth_data = json.loads(f.read()) + uuid = auth_data['uuid'] + server_url = auth_data['server_url'] + cert_file = auth_data['cert_file'] + token = auth_data['token'] + + # export auth data to a file + if args.export_auth_data: + with open(args.export_auth_data, "w") as f: + f.write(json.dumps({ + 'uuid': uuid, + 'server_url': server_url, + 'cert_file': cert_file, + 'token': token, + })) + + soledad = _get_soledad_instance( + uuid, passphrase, basedir, server_url, cert_file, token) + km = _get_keymanager_instance( + args.username, + args.provider, + soledad, + token, + uid=uuid) + _main(soledad, km, args) + reactor.run() diff --git a/scripts/docker/files/start-client-test.sh b/scripts/docker/files/start-client-test.sh new file mode 100755 index 00000000..1275b50d --- /dev/null +++ b/scripts/docker/files/start-client-test.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Run Soledad tests. + +CMD="/usr/local/soledad/test-env.py" +REPO="/var/local/soledad" + +if [ ! -z "${SOLEDAD_REMOTE}" ]; then + git -C ${REPO} remote add test ${SOLEDAD_REMOTE} + git -C ${REPO} fetch test +fi + +if [ ! -z "${SOLEDAD_BRANCH}" ]; then + git -C ${REPO} checkout ${SOLEDAD_BRANCH} +fi + +${CMD} soledad-client test --server-url ${SOLEDAD_SERVER_URL} diff --git a/scripts/docker/files/util.py b/scripts/docker/files/util.py new file mode 100644 index 00000000..e7e2ef9a --- /dev/null +++ b/scripts/docker/files/util.py @@ -0,0 +1,75 @@ +import re +import psutil +import time +import threading +import argparse +import pytz +import datetime + + +class ValidateUserHandle(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + m = re.compile('^([^@]+)@([^@]+\.[^@]+)$') + res = m.match(values) + if res == None: + parser.error('User handle should have the form user@provider.') + setattr(namespace, 'username', res.groups()[0]) + setattr(namespace, 'provider', res.groups()[1]) + + +class StatsLogger(threading.Thread): + + def __init__(self, name, fname, procs=[], interval=0.01): + threading.Thread.__init__(self) + self._stopped = True + self._name = name + self._fname = fname + self._procs = self._find_procs(procs) + self._interval = interval + + def _find_procs(self, procs): + return filter(lambda p: p.name in procs, psutil.process_iter()) + + def run(self): + self._stopped = False + with open(self._fname, 'w') as f: + self._start = time.time() + f.write(self._make_header()) + while self._stopped is False: + f.write('%s %s\n' % + (self._make_general_stats(), self._make_proc_stats())) + time.sleep(self._interval) + f.write(self._make_footer()) + + def _make_general_stats(self): + now = time.time() + stats = [] + stats.append("%f" % (now - self._start)) # elapsed time + stats.append("%f" % psutil.cpu_percent()) # total cpu + stats.append("%f" % psutil.virtual_memory().percent) # total memory + return ' '.join(stats) + + def _make_proc_stats(self): + stats = [] + for p in self._procs: + stats.append('%f' % p.get_cpu_percent()) # proc cpu + stats.append('%f' % p.get_memory_percent()) # proc memory + return ' '.join(stats) + + def _make_header(self): + header = [] + header.append('# test_name: %s' % self._name) + header.append('# start_time: %s' % datetime.datetime.now(pytz.utc)) + header.append( + '# elapsed_time total_cpu total_memory proc_cpu proc_memory ') + return '\n'.join(header) + '\n' + + def _make_footer(self): + footer = [] + footer.append('# end_time: %s' % datetime.datetime.now(pytz.utc)) + return '\n'.join(footer) + + def stop(self): + self._stopped = True + + diff --git a/scripts/docker/helper/run-tests.sh b/scripts/docker/helper/run-tests.sh new file mode 100755 index 00000000..cee90f6b --- /dev/null +++ b/scripts/docker/helper/run-tests.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +tempfile=`mktemp -u` +make run-server CONTAINER_ID_FILE=${tempfile} +sleep 5 +make run-client-test CONTAINER_ID_FILE=${tempfile} -- cgit v1.2.3 From 23410ebd3e2fbf6b2a803ea7e437bf51a2481fef Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 6 Jun 2016 18:06:53 -0300 Subject: [test] fix docker makefile target for running client test --- scripts/docker/Makefile | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/scripts/docker/Makefile b/scripts/docker/Makefile index 7a80fe02..b9432e8a 100644 --- a/scripts/docker/Makefile +++ b/scripts/docker/Makefile @@ -17,14 +17,12 @@ run-server: image $(IMAGE_NAME) \ /usr/local/soledad/start-server.sh -# TODO: the following rule does not work for now, we have to add a -# `start-test.sh` file -run-test: image +run-client-test: image container_id=`cat $(CONTAINER_ID_FILE)`; \ server_ip=`./helper/get-container-ip.sh $${container_id}`; \ - docker run \ + docker run -t -i \ --env="SOLEDAD_REMOTE=https://0xacab.org/leap/soledad.git" \ --env="SOLEDAD_BRANCH=develop" \ - --env="SOLEDAD_SERVER_IP=$${server_ip}" \ + --env="SOLEDAD_SERVER_URL=http://$${server_ip}:2424" \ $(IMAGE_NAME) \ - /usr/local/soledad/start-test.sh + /usr/local/soledad/start-client-test.sh -- cgit v1.2.3 From afa26b8de6e900e3022528ee7ea37b504c19688b Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 6 Jun 2016 18:07:31 -0300 Subject: [test] fix test-env script command line option --- scripts/docker/files/test-env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/docker/files/test-env.py b/scripts/docker/files/test-env.py index 6ff0a4ba..41784464 100755 --- a/scripts/docker/files/test-env.py +++ b/scripts/docker/files/test-env.py @@ -623,7 +623,7 @@ def run_cli(): (['--uuid', '-u'], {'default': SOLEDAD_CLIENT_UUID}), (['--passphrase', '-k'], {'default': SOLEDAD_CLIENT_PASS}), (['--basedir', '-b'], {'default': SOLEDAD_CLIENT_BASEDIR}), - (['--server_url', '-s'], {'default': SOLEDAD_SERVER_URL}), + (['--server-url', '-s'], {'default': SOLEDAD_SERVER_URL}), (['--cert-key', '-C'], {'default': os.path.join( SOLEDAD_SERVER_BASEDIR, SOLEDAD_SERVER_CERTKEY)}), -- cgit v1.2.3 From 8b7f2419cd8610e30ce6fd059f3c703cc2257af6 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 6 Jun 2016 18:07:53 -0300 Subject: [test] add soledad-perf repo to docker image --- scripts/docker/files/setup-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/docker/files/setup-env.sh b/scripts/docker/files/setup-env.sh index c98a6d08..1df1c825 100755 --- a/scripts/docker/files/setup-env.sh +++ b/scripts/docker/files/setup-env.sh @@ -12,7 +12,7 @@ BASEURL="https://github.com/leapcode" mkdir -p ${BASEDIR} # clone repositories -repos="soledad leap_pycommon" +repos="soledad leap_pycommon soledad-perf" for repo in ${repos}; do repodir=${BASEDIR}/${repo} -- cgit v1.2.3 From def78c4d33e0a4169f90866eae5f0bb3e013041f Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 6 Jun 2016 18:08:23 -0300 Subject: [test] fix docker image run example in readme file --- scripts/docker/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/docker/README.md b/scripts/docker/README.md index d15129fa..101db837 100644 --- a/scripts/docker/README.md +++ b/scripts/docker/README.md @@ -27,4 +27,7 @@ a specific branch on the soledad repository: Example: - docker run leap/soledad:1.0 /usr/local/soledad/start-server.sh + docker run \ + --env="SOLEDAD_REMOTE=https://0xacab.org/leap/soledad.git" \ + --env="SOLEDAD_BRANCH=develop" \ + leap/soledad:1.0 /usr/local/soledad/start-server.sh -- cgit v1.2.3 From 07b9ebf33a6b4499d0ad4845bec1e2836dbbeede Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 6 Jun 2016 18:08:39 -0300 Subject: [test] pep8 fix on test script --- scripts/db_access/client_side_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/db_access/client_side_db.py b/scripts/db_access/client_side_db.py index 25eebfbe..11d72791 100644 --- a/scripts/db_access/client_side_db.py +++ b/scripts/db_access/client_side_db.py @@ -55,7 +55,7 @@ def _fail(reason): def _get_api_info(provider): info = requests.get( - 'https://'+provider+'/provider.json', verify=False).json() + 'https://' + provider + '/provider.json', verify=False).json() return info['api_uri'], info['api_version'] -- cgit v1.2.3 From 8823fd51ce24d0afae82541d614eeb1ef8c5c0c3 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 6 Jun 2016 18:09:39 -0300 Subject: [test] add makefile option to rm all docker containers --- scripts/docker/Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/docker/Makefile b/scripts/docker/Makefile index b9432e8a..5e5b345e 100644 --- a/scripts/docker/Makefile +++ b/scripts/docker/Makefile @@ -26,3 +26,6 @@ run-client-test: image --env="SOLEDAD_SERVER_URL=http://$${server_ip}:2424" \ $(IMAGE_NAME) \ /usr/local/soledad/start-client-test.sh + +rm-all-containers: + docker ps -a | cut -d" " -f 1 | tail -n +2 | xargs docker rm -f -- cgit v1.2.3 From f25dc033f9456ea82be5330fe9a3e5145a78f361 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 6 Jun 2016 18:10:29 -0300 Subject: [test] add shared db setup to docker start server script --- scripts/docker/files/start-server.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/docker/files/start-server.sh b/scripts/docker/files/start-server.sh index ea14aa5a..7493930a 100755 --- a/scripts/docker/files/start-server.sh +++ b/scripts/docker/files/start-server.sh @@ -21,5 +21,6 @@ ${CMD} couch start ${CMD} user-db create ${CMD} token-db create ${CMD} token-db insert-token +${CMD} shared-db create ${CMD} cert create ${CMD} soledad-server start --no-daemonize -- cgit v1.2.3 From 7ee12a38d6e8230525f3401abdbd3def4d81502b Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 6 Jun 2016 20:34:03 -0300 Subject: [test] add rule and script to run soledad connection test --- scripts/docker/Makefile | 42 +++++++++++++++++++++++++++++------- scripts/docker/helper/run-tests.sh | 44 +++++++++++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 9 deletions(-) diff --git a/scripts/docker/Makefile b/scripts/docker/Makefile index 5e5b345e..9b9ab8f7 100644 --- a/scripts/docker/Makefile +++ b/scripts/docker/Makefile @@ -1,31 +1,57 @@ #/usr/bin/env -IMAGE_NAME = "leap/soledad:1.0" +# This makefile is intended to aid on running soledad docker images for +# specific purposes, as running a server, a client or tests. +# +# In order to communicate the IP address of one container to another, we make +# use of a file containing the container id. You have to explicitelly pass the +# CONTAINER_ID_FILE variable when invoking some of the targets below. +# +# Example usage: +# +# make run-server CONTAINER_ID_FILE=/tmp/container-id.txt +# make run-client-test CONTAINER_ID_FILE=/tmp/container-id.txt + + +IMAGE_NAME ?= "leap/soledad:1.0" +SOLEDAD_REMOTE ?= "https://0xacab.org/leap/soledad.git" +SOLEDAD_BRANCH ?= "develop" + all: image image: docker build -t $(IMAGE_NAME) . -run-server: image - rm -f $(CONTAINER_ID_FILE) +run-server: + @if [ -z "$(CONTAINER_ID_FILE)" ]; then \ + echo "Error: you have to pass a value to CONTAINER_ID_FILE."; \ + exit 2; \ + fi docker run \ - --env="SOLEDAD_REMOTE=https://0xacab.org/leap/soledad.git" \ - --env="SOLEDAD_BRANCH=develop" \ + --env="SOLEDAD_REMOTE=$(SOLEDAD_REMOTE)" \ + --env="SOLEDAD_BRANCH=$(SOLEDAD_BRANCH)" \ --cidfile=$(CONTAINER_ID_FILE) \ --detach \ $(IMAGE_NAME) \ /usr/local/soledad/start-server.sh -run-client-test: image +run-client-test: + @if [ -z "$(CONTAINER_ID_FILE)" ]; then \ + echo "Error: you have to pass a value to CONTAINER_ID_FILE."; \ + exit 2; \ + fi container_id=`cat $(CONTAINER_ID_FILE)`; \ server_ip=`./helper/get-container-ip.sh $${container_id}`; \ docker run -t -i \ - --env="SOLEDAD_REMOTE=https://0xacab.org/leap/soledad.git" \ - --env="SOLEDAD_BRANCH=develop" \ + --env="SOLEDAD_REMOTE=$(SOLEDAD_REMOTE)" \ + --env="SOLEDAD_BRANCH=$(SOLEDAD_BRANCH)" \ --env="SOLEDAD_SERVER_URL=http://$${server_ip}:2424" \ $(IMAGE_NAME) \ /usr/local/soledad/start-client-test.sh +run-shell: image + docker run -t -i $(IMAGE_NAME) /bin/bash + rm-all-containers: docker ps -a | cut -d" " -f 1 | tail -n +2 | xargs docker rm -f diff --git a/scripts/docker/helper/run-tests.sh b/scripts/docker/helper/run-tests.sh index cee90f6b..bcd7a565 100755 --- a/scripts/docker/helper/run-tests.sh +++ b/scripts/docker/helper/run-tests.sh @@ -1,6 +1,48 @@ #!/bin/sh +# Run 2 docker images, one with soledad server and another with a soledad +# client running the tests. +# +# After launching the server, the script waits for TIMEOUT seconds for it to +# come up. If we fail to detect the server, the script exits with nonzero +# status. + + +# seconds to wait before giving up waiting from server +TIMEOUT=20 + +# some info from this script +SCRIPT=$(readlink -f "$0") +SCRIPTPATH=$(dirname "$SCRIPT") + +# run the server tempfile=`mktemp -u` make run-server CONTAINER_ID_FILE=${tempfile} -sleep 5 + +# get server container info +container_id=`cat ${tempfile}` +server_ip=`${SCRIPTPATH}/get-container-ip.sh ${container_id}` + +# wait for server until timeout +start=`date +%s` +elapsed=0 + +while [ ${elapsed} -lt ${TIMEOUT} ]; do + result=`curl http://${server_ip}:2424` + if [ ${?} -eq 0 ]; then + break + else + sleep 1 + fi + now=`date +%s` + elapsed=`expr ${now} - ${start}` +done + +# exit with an error code if timed out waiting for server +if [ ${elapsed} -ge ${TIMEOUT} ]; then + echo "Error: server unreacheble at ${server_ip} after ${TIMEOUT} seconds." + exit 1 +fi + +# run the client make run-client-test CONTAINER_ID_FILE=${tempfile} -- cgit v1.2.3 From 57e21dc89f1cc5e34261da75d939e53edd3d5a2b Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 6 Jun 2016 20:43:59 -0300 Subject: [test] add rule for running trial tests in a docker container --- scripts/docker/Dockerfile | 1 + scripts/docker/Makefile | 7 +++++++ scripts/docker/files/start-trial-test.sh | 19 +++++++++++++++++++ 3 files changed, 27 insertions(+) create mode 100755 scripts/docker/files/start-trial-test.sh diff --git a/scripts/docker/Dockerfile b/scripts/docker/Dockerfile index 24bfff30..8b7dcb71 100644 --- a/scripts/docker/Dockerfile +++ b/scripts/docker/Dockerfile @@ -39,4 +39,5 @@ COPY files/client_side_db.py /usr/local/soledad/ COPY files/util.py /usr/local/soledad/ COPY files/start-server.sh /usr/local/soledad/ COPY files/start-client-test.sh /usr/local/soledad/ +COPY files/start-trial-test.sh /usr/local/soledad/ COPY files/conf/* /usr/local/soledad/conf/ diff --git a/scripts/docker/Makefile b/scripts/docker/Makefile index 9b9ab8f7..872bdc40 100644 --- a/scripts/docker/Makefile +++ b/scripts/docker/Makefile @@ -50,6 +50,13 @@ run-client-test: $(IMAGE_NAME) \ /usr/local/soledad/start-client-test.sh +run-trial-test: + docker run -t -i \ + --env="SOLEDAD_REMOTE=$(SOLEDAD_REMOTE)" \ + --env="SOLEDAD_BRANCH=$(SOLEDAD_BRANCH)" \ + $(IMAGE_NAME) \ + /usr/local/soledad/start-trial-test.sh + run-shell: image docker run -t -i $(IMAGE_NAME) /bin/bash diff --git a/scripts/docker/files/start-trial-test.sh b/scripts/docker/files/start-trial-test.sh new file mode 100755 index 00000000..98b09e53 --- /dev/null +++ b/scripts/docker/files/start-trial-test.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Run Soledad tests. + +CMD="/usr/local/soledad/test-env.py" +REPO="/var/local/soledad" + +if [ ! -z "${SOLEDAD_REMOTE}" ]; then + git -C ${REPO} remote add test ${SOLEDAD_REMOTE} + git -C ${REPO} fetch test +fi + +if [ ! -z "${SOLEDAD_BRANCH}" ]; then + git -C ${REPO} checkout ${SOLEDAD_BRANCH} +fi + +${CMD} couch start + +trial leap.soledad.common -- cgit v1.2.3 From 17b3eb6967b944e8e500190d57c79978c1f9fd57 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 6 Jun 2016 21:21:04 -0300 Subject: [doc] improve docker script docs --- scripts/docker/files/setup-env.sh | 11 +++++++++++ scripts/docker/files/start-client-test.sh | 5 ++++- scripts/docker/files/start-trial-test.sh | 6 +++++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/scripts/docker/files/setup-env.sh b/scripts/docker/files/setup-env.sh index 1df1c825..d5aeab7d 100755 --- a/scripts/docker/files/setup-env.sh +++ b/scripts/docker/files/setup-env.sh @@ -3,6 +3,17 @@ # Clone soledad repository and install soledad dependencies needed to run # client and server in a test environment. # +# In details, this script does the following: +# +# - clone a series of python package repositories into /var/local/soledad. +# - install dependencies for those packages from the requirements files in +# each of the repositories, using python wheels when possible. +# - install the python packages in development mode +# +# The cloned git repositories might have a remote configured and a branch +# checked out on runtime, before a server, client or test instance is actually +# run. Check the other scripts in this directory. +# # This script is meant to be copied to the docker container and run after # system dependencies have been installed. diff --git a/scripts/docker/files/start-client-test.sh b/scripts/docker/files/start-client-test.sh index 1275b50d..5997385f 100755 --- a/scripts/docker/files/start-client-test.sh +++ b/scripts/docker/files/start-client-test.sh @@ -1,6 +1,9 @@ #!/bin/bash -# Run Soledad tests. +# Run a Soledad client connection test. +# +# This script is meant to be copied to the docker container and run upon +# container start. CMD="/usr/local/soledad/test-env.py" REPO="/var/local/soledad" diff --git a/scripts/docker/files/start-trial-test.sh b/scripts/docker/files/start-trial-test.sh index 98b09e53..15638b65 100755 --- a/scripts/docker/files/start-trial-test.sh +++ b/scripts/docker/files/start-trial-test.sh @@ -1,6 +1,9 @@ #!/bin/bash -# Run Soledad tests. +# Run Soledad trial tests. +# +# This script is meant to be copied to the docker container and run upon +# container start. CMD="/usr/local/soledad/test-env.py" REPO="/var/local/soledad" @@ -14,6 +17,7 @@ if [ ! -z "${SOLEDAD_BRANCH}" ]; then git -C ${REPO} checkout ${SOLEDAD_BRANCH} fi +# currently soledad trial tests need a running couch on environment ${CMD} couch start trial leap.soledad.common -- cgit v1.2.3 From f2739e7a100035d1d71eeffa3190b805a5931a50 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 7 Jun 2016 22:55:26 -0400 Subject: [pkg] remove unused chardet dependency --- client/pkg/requirements.pip | 1 - 1 file changed, 1 deletion(-) diff --git a/client/pkg/requirements.pip b/client/pkg/requirements.pip index 2f658d76..42c0d0b1 100644 --- a/client/pkg/requirements.pip +++ b/client/pkg/requirements.pip @@ -1,7 +1,6 @@ pysqlcipher>2.6.3 u1db scrypt -cchardet zope.proxy twisted -- cgit v1.2.3 From d11849105e06b16e03ad938b6e41e934e99a33cc Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 19 May 2016 11:12:35 -0300 Subject: [feat] add .gitlab-ci.yml --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..c6d1998c --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,3 @@ +run_tests: + script: + - /home/gitlab-runner/soledad/scripts/gitlab/run_tests.sh -- cgit v1.2.3 From f01b2cf3aa27350eae788152f95c4f9ca8e11b9f Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 19 May 2016 17:19:31 -0300 Subject: [bug] install pip from default location Old versions of pip do not accept the --trusted-host option and will complain when trying to upgrade pip from wheel. To fix that we upgrade pip from usual location instead of doing it from wheel. --- client/pkg/pip_install_requirements.sh | 2 +- common/pkg/pip_install_requirements.sh | 2 +- server/pkg/pip_install_requirements.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/pkg/pip_install_requirements.sh b/client/pkg/pip_install_requirements.sh index d0479365..1f5ac5f6 100755 --- a/client/pkg/pip_install_requirements.sh +++ b/client/pkg/pip_install_requirements.sh @@ -80,5 +80,5 @@ insecure_flags=`return_insecure_flags` packages=`return_packages` pip install -U wheel -pip install $install_options pip +pip install -U pip pip install $install_options $insecure_flags $packages diff --git a/common/pkg/pip_install_requirements.sh b/common/pkg/pip_install_requirements.sh index d0479365..1f5ac5f6 100755 --- a/common/pkg/pip_install_requirements.sh +++ b/common/pkg/pip_install_requirements.sh @@ -80,5 +80,5 @@ insecure_flags=`return_insecure_flags` packages=`return_packages` pip install -U wheel -pip install $install_options pip +pip install -U pip pip install $install_options $insecure_flags $packages diff --git a/server/pkg/pip_install_requirements.sh b/server/pkg/pip_install_requirements.sh index d0479365..1f5ac5f6 100755 --- a/server/pkg/pip_install_requirements.sh +++ b/server/pkg/pip_install_requirements.sh @@ -80,5 +80,5 @@ insecure_flags=`return_insecure_flags` packages=`return_packages` pip install -U wheel -pip install $install_options pip +pip install -U pip pip install $install_options $insecure_flags $packages -- cgit v1.2.3 From 929650db7d74011614b4c30ad1dc838f5ecbc266 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 19 May 2016 21:53:33 -0300 Subject: [feature] add script to run tests for a gitlab ci runner --- scripts/gitlab/run_soledad_tests.sh | 132 ++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 scripts/gitlab/run_soledad_tests.sh diff --git a/scripts/gitlab/run_soledad_tests.sh b/scripts/gitlab/run_soledad_tests.sh new file mode 100644 index 00000000..328fcc4d --- /dev/null +++ b/scripts/gitlab/run_soledad_tests.sh @@ -0,0 +1,132 @@ +#!/bin/sh + + +# Name: +# run_tests.sh -- run soledad tests from a given source directory +# +# Usage: +# run_tests.sh +# +# Description: +# This script sets up a minimal test environment from a soledad source +# directory and runs all tests, returning the same exit status as the test +# process. As it is intended to be called by a GitLab Runner, it expects the +# following GitLab CI predefined variable to be set in the environment: +# +# CI_PROJECT_DIR The full path where the repository is cloned and where +# the build is ran +# +# Example: +# CI_PROJECT_DIR=/tmp/soledad run_tests.sh + + +# Fail if expected variable is not a directory. +if [ ! -d "${CI_PROJECT_DIR}" ]; then + echo "Error! Not a directory: ${CI_PROJECT_DIR}" + exit 1 +fi + + +# Setup pip to use wheels because it is prebuilt and thus faster to deploy. + +PIP_INSTALL="pip install -U \ + --find-links=https://lizard.leap.se/wheels \ + --trusted-host lizard.leap.se" + + +# Use a fresh python virtual envinroment each time. + +setup_venv() { + venv=${1} + virtualenv ${venv} + . ${venv}/bin/activate +} + + +# Most of the dependencies are installed directly from soledad source pip +# requirement files. Some of them need alternative ways of installing because +# of distinct reasons, see below. + +install_deps() { + install_leap_common + install_scrypt + install_soledad_deps +} + + +# Install scrypt manually to avoid import problems as the ones described in +# https://leap.se/code/issues/4948 + +install_scrypt() { + pip install scrypt +} + + +# Iterate through soledad components and use the special pip install script to +# install (mostly) all requirements for testing. +# +# TODO: Soledad tests should depend on almost nothing and have every component +# from other leap packages mocked. + +install_soledad_deps() { + for pkg in common client server; do + testing="--testing" + if [ "${pkg}" = "server" ]; then + # soledad server doesn't currently have a requirements-testing.pip file, + # so we don't pass the option when that is the case + testing="" + fi + (cd ${CI_PROJECT_DIR}/${pkg} \ + && ./pkg/pip_install_requirements.sh ${testing} --use-leap-wheels \ + && python setup.py develop) + done +} + + +# We have to manually install leap.common from source because: +# +# - the leap.common package is not currently set as a "testing dependency" +# for soledad; and +# +# - having another package from the leap namespace installed from egg or +# wheels may confuse the python interpreter when importing modules. + +install_leap_common() { + temp=`mktemp -d` + host="git://github.com/leapcode" + proj="leap_pycommon" + git clone ${host}/${proj} ${temp}/${proj} + (cd ${temp}/${proj} \ + && ./pkg/pip_install_requirements.sh \ + && python setup.py develop) +} + + +# Run soledad tests. The exit status of the following function is used as the +# script's exit status. + +run_tests() { + trial leap.soledad.common.tests +} + + +# Cleanup leftovers before finishing. + +cleanup_venv() { + venv=${1} + rm -rf ${venv} +} + + +main() { + venv="`mktemp -d`/venv" + setup_venv ${venv} + install_deps + run_tests + status=$? + cleanup_venv ${venv} + exit ${status} +} + + +main -- cgit v1.2.3 From e94324c7a4d4e293fe3e38fa3c1becec146372e9 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 19 May 2016 22:04:48 -0300 Subject: [refactor] reorganize scripts dir --- scripts/build_debian_package.sh | 32 --------- scripts/compile_design_docs.py | 111 ------------------------------ scripts/develop_mode.sh | 7 -- scripts/packaging/build_debian_package.sh | 32 +++++++++ scripts/packaging/compile_design_docs.py | 111 ++++++++++++++++++++++++++++++ scripts/run_tests.sh | 3 - scripts/testing/develop_mode.sh | 7 ++ 7 files changed, 150 insertions(+), 153 deletions(-) delete mode 100755 scripts/build_debian_package.sh delete mode 100644 scripts/compile_design_docs.py delete mode 100755 scripts/develop_mode.sh create mode 100755 scripts/packaging/build_debian_package.sh create mode 100644 scripts/packaging/compile_design_docs.py delete mode 100755 scripts/run_tests.sh create mode 100755 scripts/testing/develop_mode.sh diff --git a/scripts/build_debian_package.sh b/scripts/build_debian_package.sh deleted file mode 100755 index b9fb93a9..00000000 --- a/scripts/build_debian_package.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh - -# This script generates Soledad Debian packages. -# -# When invoking this script, you should pass a git repository URL and the name -# of the branch that contains the code you wish to build the packages from. -# -# The script will clone the given branch from the given repo, as well as the -# main Soledad repo in github which contains the most up-to-date debian -# branch. It will then merge the desired branch into the debian branch and -# build the packages. - -if [ $# -ne 2 ]; then - echo "Usage: ${0} " - exit 1 -fi - -SOLEDAD_MAIN_REPO=git://github.com/leapcode/soledad.git - -url=$1 -branch=$2 -workdir=`mktemp -d` - -git clone -b ${branch} ${url} ${workdir}/soledad -export GIT_DIR=${workdir}/soledad/.git -export GIT_WORK_TREE=${workdir}/soledad -git remote add leapcode ${SOLEDAD_MAIN_REPO} -git fetch leapcode -git checkout -b debian/experimental leapcode/debian/experimental -git merge --no-edit ${branch} -(cd ${workdir}/soledad && debuild -uc -us) -echo "Packages generated in ${workdir}" diff --git a/scripts/compile_design_docs.py b/scripts/compile_design_docs.py deleted file mode 100644 index 7ffebb10..00000000 --- a/scripts/compile_design_docs.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/python - - -# This script builds files for the design documents represented in the -# ../common/src/soledad/common/ddocs directory structure (relative to the -# current location of the script) into a target directory. - - -import argparse -from os import listdir -from os.path import realpath, dirname, isdir, join, isfile, basename -import json - -DDOCS_REL_PATH = ('..', 'common', 'src', 'leap', 'soledad', 'common', 'ddocs') - - -def build_ddocs(): - """ - Build design documents. - - For ease of development, couch backend design documents are stored as - `.js` files in subdirectories of - `../common/src/leap/soledad/common/ddocs`. This function scans that - directory for javascript files, and builds the design documents structure. - - This funciton uses the following conventions to generate design documents: - - - Design documents are represented by directories in the form - `/`, there prefix is the `src/leap/soledad/common/ddocs` - directory. - - Design document directories might contain `views`, `lists` and - `updates` subdirectories. - - Views subdirectories must contain a `map.js` file and may contain a - `reduce.js` file. - - List and updates subdirectories may contain any number of javascript - files (i.e. ending in `.js`) whose names will be mapped to the - corresponding list or update function name. - """ - ddocs = {} - - # design docs are represented by subdirectories of `DDOCS_REL_PATH` - cur_pwd = dirname(realpath(__file__)) - ddocs_path = join(cur_pwd, *DDOCS_REL_PATH) - for ddoc in [f for f in listdir(ddocs_path) - if isdir(join(ddocs_path, f))]: - - ddocs[ddoc] = {'_id': '_design/%s' % ddoc} - - for t in ['views', 'lists', 'updates']: - tdir = join(ddocs_path, ddoc, t) - if isdir(tdir): - - ddocs[ddoc][t] = {} - - if t == 'views': # handle views (with map/reduce functions) - for view in [f for f in listdir(tdir) - if isdir(join(tdir, f))]: - # look for map.js and reduce.js - mapfile = join(tdir, view, 'map.js') - reducefile = join(tdir, view, 'reduce.js') - mapfun = None - reducefun = None - try: - with open(mapfile) as f: - mapfun = f.read() - except IOError: - pass - try: - with open(reducefile) as f: - reducefun = f.read() - except IOError: - pass - ddocs[ddoc]['views'][view] = {} - - if mapfun is not None: - ddocs[ddoc]['views'][view]['map'] = mapfun - if reducefun is not None: - ddocs[ddoc]['views'][view]['reduce'] = reducefun - - else: # handle lists, updates, etc - for fun in [f for f in listdir(tdir) - if isfile(join(tdir, f))]: - funfile = join(tdir, fun) - funname = basename(funfile).replace('.js', '') - try: - with open(funfile) as f: - ddocs[ddoc][t][funname] = f.read() - except IOError: - pass - return ddocs - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument( - 'target', type=str, - help='the target dir where to store design documents') - args = parser.parse_args() - - # check if given target is a directory - if not isdir(args.target): - print 'Error: %s is not a directory.' % args.target - exit(1) - - # write desifgn docs files - ddocs = build_ddocs() - for ddoc in ddocs: - ddoc_filename = "%s.json" % ddoc - with open(join(args.target, ddoc_filename), 'w') as f: - f.write("%s" % json.dumps(ddocs[ddoc], indent=3)) - print "Wrote _design/%s content in %s" % (ddoc, join(args.target, ddoc_filename,)) diff --git a/scripts/develop_mode.sh b/scripts/develop_mode.sh deleted file mode 100755 index 8d2ebaa8..00000000 --- a/scripts/develop_mode.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -cd common -python setup.py develop -cd ../client -python setup.py develop -cd ../server -python setup.py develop diff --git a/scripts/packaging/build_debian_package.sh b/scripts/packaging/build_debian_package.sh new file mode 100755 index 00000000..b9fb93a9 --- /dev/null +++ b/scripts/packaging/build_debian_package.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +# This script generates Soledad Debian packages. +# +# When invoking this script, you should pass a git repository URL and the name +# of the branch that contains the code you wish to build the packages from. +# +# The script will clone the given branch from the given repo, as well as the +# main Soledad repo in github which contains the most up-to-date debian +# branch. It will then merge the desired branch into the debian branch and +# build the packages. + +if [ $# -ne 2 ]; then + echo "Usage: ${0} " + exit 1 +fi + +SOLEDAD_MAIN_REPO=git://github.com/leapcode/soledad.git + +url=$1 +branch=$2 +workdir=`mktemp -d` + +git clone -b ${branch} ${url} ${workdir}/soledad +export GIT_DIR=${workdir}/soledad/.git +export GIT_WORK_TREE=${workdir}/soledad +git remote add leapcode ${SOLEDAD_MAIN_REPO} +git fetch leapcode +git checkout -b debian/experimental leapcode/debian/experimental +git merge --no-edit ${branch} +(cd ${workdir}/soledad && debuild -uc -us) +echo "Packages generated in ${workdir}" diff --git a/scripts/packaging/compile_design_docs.py b/scripts/packaging/compile_design_docs.py new file mode 100644 index 00000000..7ffebb10 --- /dev/null +++ b/scripts/packaging/compile_design_docs.py @@ -0,0 +1,111 @@ +#!/usr/bin/python + + +# This script builds files for the design documents represented in the +# ../common/src/soledad/common/ddocs directory structure (relative to the +# current location of the script) into a target directory. + + +import argparse +from os import listdir +from os.path import realpath, dirname, isdir, join, isfile, basename +import json + +DDOCS_REL_PATH = ('..', 'common', 'src', 'leap', 'soledad', 'common', 'ddocs') + + +def build_ddocs(): + """ + Build design documents. + + For ease of development, couch backend design documents are stored as + `.js` files in subdirectories of + `../common/src/leap/soledad/common/ddocs`. This function scans that + directory for javascript files, and builds the design documents structure. + + This funciton uses the following conventions to generate design documents: + + - Design documents are represented by directories in the form + `/`, there prefix is the `src/leap/soledad/common/ddocs` + directory. + - Design document directories might contain `views`, `lists` and + `updates` subdirectories. + - Views subdirectories must contain a `map.js` file and may contain a + `reduce.js` file. + - List and updates subdirectories may contain any number of javascript + files (i.e. ending in `.js`) whose names will be mapped to the + corresponding list or update function name. + """ + ddocs = {} + + # design docs are represented by subdirectories of `DDOCS_REL_PATH` + cur_pwd = dirname(realpath(__file__)) + ddocs_path = join(cur_pwd, *DDOCS_REL_PATH) + for ddoc in [f for f in listdir(ddocs_path) + if isdir(join(ddocs_path, f))]: + + ddocs[ddoc] = {'_id': '_design/%s' % ddoc} + + for t in ['views', 'lists', 'updates']: + tdir = join(ddocs_path, ddoc, t) + if isdir(tdir): + + ddocs[ddoc][t] = {} + + if t == 'views': # handle views (with map/reduce functions) + for view in [f for f in listdir(tdir) + if isdir(join(tdir, f))]: + # look for map.js and reduce.js + mapfile = join(tdir, view, 'map.js') + reducefile = join(tdir, view, 'reduce.js') + mapfun = None + reducefun = None + try: + with open(mapfile) as f: + mapfun = f.read() + except IOError: + pass + try: + with open(reducefile) as f: + reducefun = f.read() + except IOError: + pass + ddocs[ddoc]['views'][view] = {} + + if mapfun is not None: + ddocs[ddoc]['views'][view]['map'] = mapfun + if reducefun is not None: + ddocs[ddoc]['views'][view]['reduce'] = reducefun + + else: # handle lists, updates, etc + for fun in [f for f in listdir(tdir) + if isfile(join(tdir, f))]: + funfile = join(tdir, fun) + funname = basename(funfile).replace('.js', '') + try: + with open(funfile) as f: + ddocs[ddoc][t][funname] = f.read() + except IOError: + pass + return ddocs + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument( + 'target', type=str, + help='the target dir where to store design documents') + args = parser.parse_args() + + # check if given target is a directory + if not isdir(args.target): + print 'Error: %s is not a directory.' % args.target + exit(1) + + # write desifgn docs files + ddocs = build_ddocs() + for ddoc in ddocs: + ddoc_filename = "%s.json" % ddoc + with open(join(args.target, ddoc_filename), 'w') as f: + f.write("%s" % json.dumps(ddocs[ddoc], indent=3)) + print "Wrote _design/%s content in %s" % (ddoc, join(args.target, ddoc_filename,)) diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh deleted file mode 100755 index e36466f8..00000000 --- a/scripts/run_tests.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -cd common -python setup.py test diff --git a/scripts/testing/develop_mode.sh b/scripts/testing/develop_mode.sh new file mode 100755 index 00000000..8d2ebaa8 --- /dev/null +++ b/scripts/testing/develop_mode.sh @@ -0,0 +1,7 @@ +#!/bin/sh +cd common +python setup.py develop +cd ../client +python setup.py develop +cd ../server +python setup.py develop -- cgit v1.2.3 From 532917f5248d0149497d6dcebfd2a1386daaff94 Mon Sep 17 00:00:00 2001 From: Victor Shyba Date: Tue, 23 Feb 2016 17:50:40 -0300 Subject: [tests] avoid using get_all_docs on asserts EncryptedSyncTestCase.test_sync_very_large_files is still getting an excessive amount of memory on very slow machines (specially on old spinning magnetic disks). This commit checks each doc at a time instead of getting them all. More refinement is necessary for this test to pass on any machine. --- .../src/leap/soledad/common/tests/test_server.py | 35 +++++++++++----------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py index ba7edfe3..328090ee 100644 --- a/common/src/leap/soledad/common/tests/test_server.py +++ b/common/src/leap/soledad/common/tests/test_server.py @@ -369,6 +369,9 @@ class EncryptedSyncTestCase( self.startTwistedServer() user = 'user-' + uuid4().hex + # this will store all docs ids to avoid get_all_docs + created_ids = [] + # instantiate soledad and create a document sol1 = self._soledad_instance( user=user, @@ -399,34 +402,32 @@ class EncryptedSyncTestCase( deferreds = [] for i in xrange(number_of_docs): content = binascii.hexlify(os.urandom(doc_size / 2)) - deferreds.append(sol1.create_doc({'data': content})) + d = sol1.create_doc({'data': content}) + d.addCallback(created_ids.append) + deferreds.append(d) return defer.DeferredList(deferreds) def _db1AssertDocsSyncedToServer(results): - _, sol_doclist = results - self.assertEqual(number_of_docs, len(sol_doclist)) - # assert doc was sent to couch db - _, couch_doclist = db.get_all_docs() - self.assertEqual(number_of_docs, len(couch_doclist)) - for i in xrange(number_of_docs): - soldoc = sol_doclist.pop() - couchdoc = couch_doclist.pop() + self.assertEqual(number_of_docs, len(created_ids)) + for soldoc in created_ids: + couchdoc = db.get_doc(soldoc.doc_id) + self.assertTrue(couchdoc) # assert document structure in couch server self.assertEqual(soldoc.doc_id, couchdoc.doc_id) self.assertEqual(soldoc.rev, couchdoc.rev) - self.assertEqual(6, len(couchdoc.content)) - self.assertTrue(crypto.ENC_JSON_KEY in couchdoc.content) - self.assertTrue(crypto.ENC_SCHEME_KEY in couchdoc.content) - self.assertTrue(crypto.ENC_METHOD_KEY in couchdoc.content) - self.assertTrue(crypto.ENC_IV_KEY in couchdoc.content) - self.assertTrue(crypto.MAC_KEY in couchdoc.content) - self.assertTrue(crypto.MAC_METHOD_KEY in couchdoc.content) + couch_content = couchdoc.content.keys() + self.assertEqual(6, len(couch_content)) + self.assertTrue(crypto.ENC_JSON_KEY in couch_content) + self.assertTrue(crypto.ENC_SCHEME_KEY in couch_content) + self.assertTrue(crypto.ENC_METHOD_KEY in couch_content) + self.assertTrue(crypto.ENC_IV_KEY in couch_content) + self.assertTrue(crypto.MAC_KEY in couch_content) + self.assertTrue(crypto.MAC_METHOD_KEY in couch_content) d = sol1.get_all_docs() d.addCallback(_db1AssertEmptyDocList) d.addCallback(_db1CreateDocs) d.addCallback(lambda _: sol1.sync()) - d.addCallback(lambda _: sol1.get_all_docs()) d.addCallback(_db1AssertDocsSyncedToServer) def _db2AssertEmptyDocList(results): -- cgit v1.2.3 From 48ff88a7781165b98285d6c25ec5d49d49cc3503 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 15 Jun 2016 17:01:38 -0400 Subject: [bug] initialize OpenSSL context just once Do not initialize the openssl context on each call to decrypt. I'm not 100% sure of the causal chain, but it seems that the initialization of the osrandom engine that openssl backend does might be breaking havoc when sqlcipher is calling rand_bytes concurrently. further testing is needed to confirm this is the ultimate cause, but in my tests this change avoids the occurrence of the dreaded Floating Point Exception in soledad/sqlcipher. - Resolves: #8180 --- client/src/leap/soledad/client/crypto.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client/src/leap/soledad/client/crypto.py b/client/src/leap/soledad/client/crypto.py index b75d4301..f7d92372 100644 --- a/client/src/leap/soledad/client/crypto.py +++ b/client/src/leap/soledad/client/crypto.py @@ -39,6 +39,8 @@ logger = logging.getLogger(__name__) MAC_KEY_LENGTH = 64 +crypto_backend = MultiBackend([OpenSSLBackend()]) + def encrypt_sym(data, key): """ @@ -59,8 +61,7 @@ def encrypt_sym(data, key): (len(key) * 8)) iv = os.urandom(16) - backend = MultiBackend([OpenSSLBackend()]) - cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend) + cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=crypto_backend) encryptor = cipher.encryptor() ciphertext = encryptor.update(data) + encryptor.finalize() @@ -87,9 +88,8 @@ def decrypt_sym(data, key, iv): soledad_assert( len(key) == 32, # 32 x 8 = 256 bits. 'Wrong key size: %s (must be 256 bits long).' % len(key)) - backend = MultiBackend([OpenSSLBackend()]) iv = binascii.a2b_base64(iv) - cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=backend) + cipher = Cipher(algorithms.AES(key), modes.CTR(iv), backend=crypto_backend) decryptor = cipher.decryptor() return decryptor.update(data) + decryptor.finalize() -- cgit v1.2.3 From a841230aa7d199151ffe1cb21d33b9b0a7bd5eb5 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 15 Jun 2016 17:29:30 -0400 Subject: [bug] move the decryption to a threadpool too --- client/src/leap/soledad/client/encdecpool.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/client/src/leap/soledad/client/encdecpool.py b/client/src/leap/soledad/client/encdecpool.py index e348f545..7e807dcf 100644 --- a/client/src/leap/soledad/client/encdecpool.py +++ b/client/src/leap/soledad/client/encdecpool.py @@ -178,9 +178,11 @@ class SyncEncrypterPool(SyncEncryptDecryptPool): secret = self._crypto.secret args = doc.doc_id, doc.rev, docstr, key, secret # encrypt asynchronously + # TODO use dedicated threadpool / move to ampoule d = threads.deferToThread( encrypt_doc_task, *args) d.addCallback(self._encrypt_doc_cb) + return d def _encrypt_doc_cb(self, result): """ @@ -429,9 +431,12 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): secret = self._crypto.secret args = doc_id, doc_rev, content, gen, trans_id, key, secret, idx # decrypt asynchronously - doc = decrypt_doc_task(*args) + # TODO use dedicated threadpool / move to ampoule + d = threads.deferToThread( + decrypt_doc_task, *args) # callback will insert it for later processing - return self._decrypt_doc_cb(doc) + d.addCallback(self._decrypt_doc_cb) + return d def insert_received_doc( self, doc_id, doc_rev, content, gen, trans_id, idx): -- cgit v1.2.3 From ab37460772c3cf07c6915baf42a61a44156cfde2 Mon Sep 17 00:00:00 2001 From: NavaL Date: Mon, 20 Jun 2016 15:03:59 +0200 Subject: [style] pep8 compatibility: indent and white space It was breaking E126 and E202 before --- client/src/leap/soledad/client/adbapi.py | 2 +- common/src/leap/soledad/common/tests/test_server.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/src/leap/soledad/client/adbapi.py b/client/src/leap/soledad/client/adbapi.py index f43e8110..cfd7675c 100644 --- a/client/src/leap/soledad/client/adbapi.py +++ b/client/src/leap/soledad/client/adbapi.py @@ -204,7 +204,7 @@ class U1DBConnectionPool(adbapi.ConnectionPool): :rtype: twisted.internet.defer.Deferred """ meth = "u1db_%s" % meth - semaphore = DeferredSemaphore(SQLCIPHER_MAX_RETRIES ) + semaphore = DeferredSemaphore(SQLCIPHER_MAX_RETRIES) def _run_interaction(): return self.runInteraction( diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py index 328090ee..2fee119d 100644 --- a/common/src/leap/soledad/common/tests/test_server.py +++ b/common/src/leap/soledad/common/tests/test_server.py @@ -482,8 +482,8 @@ class EncryptedSyncTestCase( Test if Soledad can sync very large files. """ self.skipTest( - "Work in progress. For reference, see: " - "https://leap.se/code/issues/7370") + "Work in progress. For reference, see: " + "https://leap.se/code/issues/7370") length = 100 * (10 ** 6) # 100 MB return self._test_encrypted_sym_sync(doc_size=length, number_of_docs=1) -- cgit v1.2.3 From f3adea709a3e33451977b1807fd2dda207af2743 Mon Sep 17 00:00:00 2001 From: NavaL Date: Mon, 20 Jun 2016 18:30:44 +0200 Subject: [bug] fix test processing order This moves the reactor time to the loopingcall period. This is necessary as the decryption is now deferred to a thread. The test will exit before the task is executed otherwise. --- common/src/leap/soledad/common/tests/test_encdecpool.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/common/src/leap/soledad/common/tests/test_encdecpool.py b/common/src/leap/soledad/common/tests/test_encdecpool.py index 9d98f44d..e6ad66ca 100644 --- a/common/src/leap/soledad/common/tests/test_encdecpool.py +++ b/common/src/leap/soledad/common/tests/test_encdecpool.py @@ -29,6 +29,7 @@ from leap.soledad.client.encdecpool import SyncDecrypterPool from leap.soledad.common.document import SoledadDocument from leap.soledad.common.tests.util import BaseSoledadTest from twisted.internet import defer +from twisted.test.proto_helpers import MemoryReactorClock DOC_ID = "mydoc" DOC_REV = "rev" @@ -217,7 +218,11 @@ class TestSyncDecrypterPool(BaseSoledadTest): This test ensures that processing of documents only occur if there is a sequence in place. """ + reactor_clock = MemoryReactorClock() + self._pool._loop.clock = reactor_clock + crypto = self._soledad._crypto + docs = [] for i in xrange(1, 10): i = str(i) @@ -239,6 +244,7 @@ class TestSyncDecrypterPool(BaseSoledadTest): yield self._pool.insert_encrypted_received_doc( doc.doc_id, doc.rev, encrypted_content, 10, "trans_id", 10) + reactor_clock.advance(self._pool.DECRYPT_LOOP_PERIOD) yield self._pool._decrypt_and_recurse() self.assertEqual(3, self._pool._processed_docs) -- cgit v1.2.3 From bcf4c28e0ad5fe1c3a3c285e50ef2a097f31cca5 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 22 Jun 2016 16:42:52 +0200 Subject: [style] pep8 --- client/src/leap/soledad/client/api.py | 3 ++- client/src/leap/soledad/client/encdecpool.py | 3 ++- client/src/leap/soledad/client/secrets.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index a1588aa9..33eae2c4 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -311,7 +311,8 @@ class Soledad(object): sync_exchange_phase = 0 if getattr(self._dbsyncer, 'syncer', None): if getattr(self._dbsyncer.syncer, 'sync_exchange_phase', None): - sync_exchange_phase = self._dbsyncer.syncer.sync_exchange_phase[0] + _p = self._dbsyncer.syncer.sync_exchange_phase[0] + sync_exchange_phase = _p return sync_phase, sync_exchange_phase # diff --git a/client/src/leap/soledad/client/encdecpool.py b/client/src/leap/soledad/client/encdecpool.py index 7e807dcf..a6d49b21 100644 --- a/client/src/leap/soledad/client/encdecpool.py +++ b/client/src/leap/soledad/client/encdecpool.py @@ -378,7 +378,8 @@ class SyncDecrypterPool(SyncEncryptDecryptPool): has finished. :rtype: twisted.internet.defer.Deferred """ - ensure_sync_id_column = "ALTER TABLE %s ADD COLUMN sync_id" % self.TABLE_NAME + ensure_sync_id_column = ("ALTER TABLE %s ADD COLUMN sync_id" % + self.TABLE_NAME) d = self._runQuery(ensure_sync_id_column) def empty_received_docs(_): diff --git a/client/src/leap/soledad/client/secrets.py b/client/src/leap/soledad/client/secrets.py index c35b881e..3547a711 100644 --- a/client/src/leap/soledad/client/secrets.py +++ b/client/src/leap/soledad/client/secrets.py @@ -441,7 +441,8 @@ class SoledadSecrets(object): if secret_id not in self._secrets: try: self._secrets[secret_id] = \ - self._decrypt_storage_secret_version_1(encrypted_secret) + self._decrypt_storage_secret_version_1( + encrypted_secret) secret_count += 1 except SecretsException as e: logger.error("Failed to decrypt storage secret: %s" -- cgit v1.2.3 From 8f5259b6ce218f8ffeec57fc04cb5c2b782bd959 Mon Sep 17 00:00:00 2001 From: drebs Date: Wed, 8 Jun 2016 17:52:59 -0300 Subject: [test] add docker perf tests --- scripts/docker/Dockerfile | 15 +++- scripts/docker/Makefile | 67 ++++++++++++++++-- scripts/docker/TODO | 2 - scripts/docker/files/run-perf-test.sh | 124 ++++++++++++++++++++++++++++++++++ scripts/docker/files/start-server.sh | 57 ++++++++++++++++ scripts/docker/helper/run-test.sh | 67 ++++++++++++++++++ scripts/docker/helper/run-tests.sh | 48 ------------- 7 files changed, 323 insertions(+), 57 deletions(-) create mode 100755 scripts/docker/files/run-perf-test.sh create mode 100755 scripts/docker/helper/run-test.sh delete mode 100755 scripts/docker/helper/run-tests.sh diff --git a/scripts/docker/Dockerfile b/scripts/docker/Dockerfile index 8b7dcb71..36180633 100644 --- a/scripts/docker/Dockerfile +++ b/scripts/docker/Dockerfile @@ -22,7 +22,15 @@ RUN apt-get -y install python-srp RUN apt-get -y install python-scrypt RUN apt-get -y install leap-keymanager RUN apt-get -y install python-tz + +# soledad-perf deps +RUN pip install klein +RUN apt-get -y install gnuplot RUN apt-get -y install curl +RUN apt-get -y install httperf + +# debugging deps +RUN apt-get -y install vim RUN apt-get -y install python-ipdb # copy over files to help setup the environment and run soledad @@ -34,10 +42,11 @@ COPY files/setup-env.sh /usr/local/soledad/ RUN /usr/local/soledad/setup-env.sh # copy runtime files for running server, client, tests, etc on a container -COPY files/test-env.py /usr/local/soledad/ COPY files/client_side_db.py /usr/local/soledad/ -COPY files/util.py /usr/local/soledad/ -COPY files/start-server.sh /usr/local/soledad/ COPY files/start-client-test.sh /usr/local/soledad/ +COPY files/run-perf-test.sh /usr/local/soledad/ +COPY files/start-server.sh /usr/local/soledad/ COPY files/start-trial-test.sh /usr/local/soledad/ +COPY files/test-env.py /usr/local/soledad/ +COPY files/util.py /usr/local/soledad/ COPY files/conf/* /usr/local/soledad/conf/ diff --git a/scripts/docker/Makefile b/scripts/docker/Makefile index 872bdc40..41334142 100644 --- a/scripts/docker/Makefile +++ b/scripts/docker/Makefile @@ -12,17 +12,29 @@ # make run-server CONTAINER_ID_FILE=/tmp/container-id.txt # make run-client-test CONTAINER_ID_FILE=/tmp/container-id.txt +##################################################################### +# Some configurations you might override when calling this makefile # +##################################################################### -IMAGE_NAME ?= "leap/soledad:1.0" -SOLEDAD_REMOTE ?= "https://0xacab.org/leap/soledad.git" -SOLEDAD_BRANCH ?= "develop" +IMAGE_NAME ?= "leap/soledad:1.0" +SOLEDAD_REMOTE ?= "https://0xacab.org/leap/soledad.git" +SOLEDAD_BRANCH ?= "develop" +SOLEDAD_PRELOAD_NUM ?= "100" +SOLEDAD_PRELOAD_SIZE ?= "500" +############################################## +# Docker image generation (main make target) # +############################################## all: image image: docker build -t $(IMAGE_NAME) . +################################################## +# Run a Soledad Server inside a docker container # +################################################## + run-server: @if [ -z "$(CONTAINER_ID_FILE)" ]; then \ echo "Error: you have to pass a value to CONTAINER_ID_FILE."; \ @@ -31,6 +43,8 @@ run-server: docker run \ --env="SOLEDAD_REMOTE=$(SOLEDAD_REMOTE)" \ --env="SOLEDAD_BRANCH=$(SOLEDAD_BRANCH)" \ + --env="SOLEDAD_PRELOAD_NUM=$(SOLEDAD_PRELOAD_NUM)" \ + --env="SOLEDAD_PRELOAD_SIZE=$(SOLEDAD_PRELOAD_SIZE)" \ --cidfile=$(CONTAINER_ID_FILE) \ --detach \ $(IMAGE_NAME) \ @@ -50,6 +64,10 @@ run-client-test: $(IMAGE_NAME) \ /usr/local/soledad/start-client-test.sh +################################################# +# Run all trial tests inside a docker container # +################################################# + run-trial-test: docker run -t -i \ --env="SOLEDAD_REMOTE=$(SOLEDAD_REMOTE)" \ @@ -57,8 +75,49 @@ run-trial-test: $(IMAGE_NAME) \ /usr/local/soledad/start-trial-test.sh +############################################ +# Performance tests and graphic generation # +############################################ + +run-perf: + helper/run-test.sh perf + +run-perf-test: + @if [ -z "$(CONTAINER_ID_FILE)" ]; then \ + echo "Error: you have to pass a value to CONTAINER_ID_FILE."; \ + exit 2; \ + fi + container_id=`cat $(CONTAINER_ID_FILE)`; \ + server_ip=`./helper/get-container-ip.sh $${container_id}`; \ + docker run -t -i \ + --cidfile=$(CONTAINER_ID_FILE)-perf \ + --env="SOLEDAD_REMOTE=$(SOLEDAD_REMOTE)" \ + --env="SOLEDAD_BRANCH=$(SOLEDAD_BRANCH)" \ + --env="SOLEDAD_PERF_REMOTE=https://0xacab.org/drebs/soledad-perf.git" \ + --env="SOLEDAD_PERF_BRANCH=bug/ensure-events-server" \ + --env="SOLEDAD_PRELOAD_NUM=$(SOLEDAD_PRELOAD_NUM)" \ + --env="SOLEDAD_PRELOAD_SIZE=$(SOLEDAD_PRELOAD_SIZE)" \ + --env="SOLEDAD_STATS=1" \ + --env="SOLEDAD_SERVER_URL=http://$${server_ip}:2424" \ + $(IMAGE_NAME) \ + /usr/local/soledad/run-perf-test.sh + +cp-perf-result: + @if [ -z "$(CONTAINER_ID_FILE)" ]; then \ + echo "Error: you have to pass a value to CONTAINER_ID_FILE."; \ + exit 2; \ + fi + perf_id=`cat $(CONTAINER_ID_FILE)-perf`; \ + docker cp $${perf_id}:/var/local/soledad-perf/out/sync-stats.png /tmp/; \ + docker cp $${perf_id}:/var/local/soledad-perf/out/series.log /tmp/ + +######################## +# Other helper targets # +######################## + run-shell: image docker run -t -i $(IMAGE_NAME) /bin/bash rm-all-containers: - docker ps -a | cut -d" " -f 1 | tail -n +2 | xargs docker rm -f + containers=`docker ps -a | cut -d" " -f 1 | tail -n +2 | xargs`; \ + if [ ! -z ${containers} ]; then docker rm -f $${containers}; fi diff --git a/scripts/docker/TODO b/scripts/docker/TODO index 75d45a8e..5185d754 100644 --- a/scripts/docker/TODO +++ b/scripts/docker/TODO @@ -1,3 +1 @@ - limit resources of containers (mem and cpu) -- add a file to run tests inside container -- use server ip to run test diff --git a/scripts/docker/files/run-perf-test.sh b/scripts/docker/files/run-perf-test.sh new file mode 100755 index 00000000..80138b2a --- /dev/null +++ b/scripts/docker/files/run-perf-test.sh @@ -0,0 +1,124 @@ +#!/bin/sh + +# Start a soledad-perf test using a remote server. +# +# The script does the following: +# +# - configure a remote repository for soledad repo if SOLEDAD_REMOTE is set. +# +# - checkout a specific branch if SOLEDAD_BRANCH is set. +# +# - run the soledad-perf local twisted server that runs the client. Note +# that the actual soledad server should be running on another docker +# container. This local server is only used to measure responsiveness of +# soledad client. The script waits for the server to come up before +# continuing, or else times out after TIMEOUT seconds. +# +# - trigger the creation of documents for sync. +# +# - start the measurement of server responsiveness and sync stages. +# +# - stop the test. +# +# This script is meant to be copied to the docker container and run upon +# container start. + +CMD="/usr/local/soledad/test-env.py" +REPO="/var/local/soledad" +TIMEOUT=20 + +#----------------------------------------------------------------------------- +# configure a remote and checkout a branch +#----------------------------------------------------------------------------- + +if [ ! -z "${SOLEDAD_REMOTE}" ]; then + git -C ${REPO} remote add test ${SOLEDAD_REMOTE} + git -C ${REPO} fetch test +fi + +if [ ! -z "${SOLEDAD_BRANCH}" ]; then + git -C ${REPO} checkout ${SOLEDAD_BRANCH} +fi + +if [ ! -z "${SOLEDAD_PERF_REMOTE}" ]; then + git -C /var/local/soledad-perf remote add test ${SOLEDAD_PERF_REMOTE} + git -C /var/local/soledad-perf fetch test +fi + +if [ ! -z "${SOLEDAD_PERF_BRANCH}" ]; then + git -C /var/local/soledad-perf checkout ${SOLEDAD_PERF_BRANCH} +fi + +#----------------------------------------------------------------------------- +# write a configuration file for the perf test +#----------------------------------------------------------------------------- + +cd /var/local/soledad-perf + +cat > defaults.conf < /dev/null & +sleep 5 # wait a bit for some data points + +# run a sync and generate a graph +make trigger-sync +make trigger-stop +make graph-image diff --git a/scripts/docker/files/start-server.sh b/scripts/docker/files/start-server.sh index 7493930a..b9b5e4ad 100755 --- a/scripts/docker/files/start-server.sh +++ b/scripts/docker/files/start-server.sh @@ -2,10 +2,28 @@ # Start a soledad server inside a docker container. # +# This script will: +# +# - eventually checkout a specific branch from a specific soledad remote. +# +# - create everything a soledad server needs to run (certificate, backend +# server database, tables, etc. +# +# - eventually preload the server database with a number of documents equal +# to SOLEDAD_PRELOAD_NUM, and with payload size equal to +# SOLEDAD_PRELOAD_SIZE. +# +# - run the soledad server. +# # This script is meant to be copied to the docker container and run upon # container start. CMD="/usr/local/soledad/test-env.py" + +#--------------------------------------------------------------------------- +# eventually checkout a specific branch from a specific remote +#--------------------------------------------------------------------------- + REPO="/var/local/soledad" if [ ! -z "${SOLEDAD_REMOTE}" ]; then @@ -17,10 +35,49 @@ if [ ! -z "${SOLEDAD_BRANCH}" ]; then git -C ${REPO} checkout ${SOLEDAD_BRANCH} fi +#--------------------------------------------------------------------------- +# setup environment for running soledad server +#--------------------------------------------------------------------------- + ${CMD} couch start ${CMD} user-db create ${CMD} token-db create ${CMD} token-db insert-token ${CMD} shared-db create ${CMD} cert create + +#--------------------------------------------------------------------------- +# write a configuration file for the perf test +#--------------------------------------------------------------------------- + +if [ "${SOLEDAD_PRELOAD_NUM}" -gt 0 ]; then + cd /var/local/soledad-perf + + cat > defaults.conf < Date: Wed, 8 Jun 2016 17:56:47 -0300 Subject: [style] pep8 fix --- scripts/packaging/compile_design_docs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/packaging/compile_design_docs.py b/scripts/packaging/compile_design_docs.py index 7ffebb10..b2b5729a 100644 --- a/scripts/packaging/compile_design_docs.py +++ b/scripts/packaging/compile_design_docs.py @@ -108,4 +108,5 @@ if __name__ == '__main__': ddoc_filename = "%s.json" % ddoc with open(join(args.target, ddoc_filename), 'w') as f: f.write("%s" % json.dumps(ddocs[ddoc], indent=3)) - print "Wrote _design/%s content in %s" % (ddoc, join(args.target, ddoc_filename,)) + print "Wrote _design/%s content in %s" \ + % (ddoc, join(args.target, ddoc_filename,)) -- cgit v1.2.3 From 6c90ea9ea4ecdb58aa70a755f05a03598ce664f6 Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 10 Jun 2016 14:35:38 -0300 Subject: [test] add memory limit to docker containers --- scripts/docker/Makefile | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/scripts/docker/Makefile b/scripts/docker/Makefile index 41334142..080fd16c 100644 --- a/scripts/docker/Makefile +++ b/scripts/docker/Makefile @@ -16,11 +16,12 @@ # Some configurations you might override when calling this makefile # ##################################################################### -IMAGE_NAME ?= "leap/soledad:1.0" -SOLEDAD_REMOTE ?= "https://0xacab.org/leap/soledad.git" -SOLEDAD_BRANCH ?= "develop" -SOLEDAD_PRELOAD_NUM ?= "100" -SOLEDAD_PRELOAD_SIZE ?= "500" +IMAGE_NAME ?= leap/soledad:1.0 +SOLEDAD_REMOTE ?= https://0xacab.org/leap/soledad.git +SOLEDAD_BRANCH ?= develop +SOLEDAD_PRELOAD_NUM ?= 100 +SOLEDAD_PRELOAD_SIZE ?= 500 +MEMORY ?= 512m ############################################## # Docker image generation (main make target) # @@ -41,6 +42,7 @@ run-server: exit 2; \ fi docker run \ + --memory="$(MEMORY)" \ --env="SOLEDAD_REMOTE=$(SOLEDAD_REMOTE)" \ --env="SOLEDAD_BRANCH=$(SOLEDAD_BRANCH)" \ --env="SOLEDAD_PRELOAD_NUM=$(SOLEDAD_PRELOAD_NUM)" \ @@ -58,6 +60,7 @@ run-client-test: container_id=`cat $(CONTAINER_ID_FILE)`; \ server_ip=`./helper/get-container-ip.sh $${container_id}`; \ docker run -t -i \ + --memory="$(MEMORY)" \ --env="SOLEDAD_REMOTE=$(SOLEDAD_REMOTE)" \ --env="SOLEDAD_BRANCH=$(SOLEDAD_BRANCH)" \ --env="SOLEDAD_SERVER_URL=http://$${server_ip}:2424" \ @@ -70,6 +73,7 @@ run-client-test: run-trial-test: docker run -t -i \ + --memory="$(MEMORY)" \ --env="SOLEDAD_REMOTE=$(SOLEDAD_REMOTE)" \ --env="SOLEDAD_BRANCH=$(SOLEDAD_BRANCH)" \ $(IMAGE_NAME) \ @@ -90,6 +94,7 @@ run-perf-test: container_id=`cat $(CONTAINER_ID_FILE)`; \ server_ip=`./helper/get-container-ip.sh $${container_id}`; \ docker run -t -i \ + --memory="$(MEMORY)" \ --cidfile=$(CONTAINER_ID_FILE)-perf \ --env="SOLEDAD_REMOTE=$(SOLEDAD_REMOTE)" \ --env="SOLEDAD_BRANCH=$(SOLEDAD_BRANCH)" \ @@ -116,8 +121,11 @@ cp-perf-result: ######################## run-shell: image - docker run -t -i $(IMAGE_NAME) /bin/bash + docker run -t -i \ + --memory="$(MEMORY)" \ + $(IMAGE_NAME) \ + /bin/bash rm-all-containers: containers=`docker ps -a | cut -d" " -f 1 | tail -n +2 | xargs`; \ - if [ ! -z ${containers} ]; then docker rm -f $${containers}; fi + if [ ! -z "$${containers}" ]; then docker rm -f $${containers}; fi -- cgit v1.2.3 From 5703d30d1789caa5705ed66f98a5b2f2dda78475 Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 10 Jun 2016 14:38:10 -0300 Subject: [test] update docker image before test --- scripts/docker/helper/run-test.sh | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/scripts/docker/helper/run-test.sh b/scripts/docker/helper/run-test.sh index 5fd6e4f0..0a2d6b6b 100755 --- a/scripts/docker/helper/run-test.sh +++ b/scripts/docker/helper/run-test.sh @@ -20,11 +20,14 @@ if [ ${#} -ne 1 ]; then exit 1 fi -TEST=${1} +test=${1} + +# make sure the image is up to date +make image # get script name and path -SCRIPT=$(readlink -f "$0") -SCRIPTPATH=$(dirname "$SCRIPT") +script=$(readlink -f "$0") +scriptpath=$(dirname "${script}") # run the server tempfile=`mktemp -u` @@ -32,7 +35,7 @@ make run-server CONTAINER_ID_FILE=${tempfile} # get server container info container_id=`cat ${tempfile}` -server_ip=`${SCRIPTPATH}/get-container-ip.sh ${container_id}` +server_ip=`${scriptpath}/get-container-ip.sh ${container_id}` # wait for server until timeout start=`date +%s` @@ -41,7 +44,7 @@ elapsed=0 echo "Waiting for soledad server container to come up..." while [ ${elapsed} -lt ${TIMEOUT} ]; do - result=`curl -s http://${server_ip}:2424` + curl -s http://${server_ip}:2424 > /dev/null if [ ${?} -eq 0 ]; then echo "Soledad server container is up!" break @@ -58,10 +61,12 @@ if [ ${elapsed} -ge ${TIMEOUT} ]; then exit 1 fi +set -e + # run the client -if [ "${TEST}" = "connect" ]; then +if [ "${test}" = "connect" ]; then make run-client-test CONTAINER_ID_FILE=${tempfile} -elif [ "${TEST}" = "perf" ]; then +elif [ "${test}" = "perf" ]; then make run-perf-test CONTAINER_ID_FILE=${tempfile} make cp-perf-result CONTAINER_ID_FILE=${tempfile} fi -- cgit v1.2.3 From 058fe53b02e73316359784abc57fb8d570da943b Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 10 Jun 2016 14:39:18 -0300 Subject: [test] create dir before cert on setup-env script --- scripts/docker/files/test-env.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/docker/files/test-env.py b/scripts/docker/files/test-env.py index 41784464..0569b65d 100755 --- a/scripts/docker/files/test-env.py +++ b/scripts/docker/files/test-env.py @@ -371,6 +371,7 @@ CERT_CONFIG_FILE = os.path.join( def cert_create(args): private_key = os.path.join(args.basedir, args.private_key) cert_key = os.path.join(args.basedir, args.cert_key) + os.mkdir(args.basedir) call([ 'openssl', 'req', -- cgit v1.2.3 From 449aaefa3a95df6f5700b645d6113458f7fde015 Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 10 Jun 2016 14:40:51 -0300 Subject: [test] improve repo config and checkout on docker scripts --- scripts/docker/files/run-perf-test.sh | 12 ++++++------ scripts/docker/files/start-client-test.sh | 4 ++-- scripts/docker/files/start-server.sh | 5 +++-- scripts/docker/files/start-trial-test.sh | 4 ++-- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/scripts/docker/files/run-perf-test.sh b/scripts/docker/files/run-perf-test.sh index 80138b2a..ebd54d23 100755 --- a/scripts/docker/files/run-perf-test.sh +++ b/scripts/docker/files/run-perf-test.sh @@ -32,8 +32,8 @@ TIMEOUT=20 #----------------------------------------------------------------------------- if [ ! -z "${SOLEDAD_REMOTE}" ]; then - git -C ${REPO} remote add test ${SOLEDAD_REMOTE} - git -C ${REPO} fetch test + git -C ${REPO} remote set-url origin ${SOLEDAD_REMOTE} + git -C ${REPO} fetch origin fi if [ ! -z "${SOLEDAD_BRANCH}" ]; then @@ -41,8 +41,8 @@ if [ ! -z "${SOLEDAD_BRANCH}" ]; then fi if [ ! -z "${SOLEDAD_PERF_REMOTE}" ]; then - git -C /var/local/soledad-perf remote add test ${SOLEDAD_PERF_REMOTE} - git -C /var/local/soledad-perf fetch test + git -C /var/local/soledad-perf remote set-url origin ${SOLEDAD_PERF_REMOTE} + git -C /var/local/soledad-perf fetch origin fi if [ ! -z "${SOLEDAD_PERF_BRANCH}" ]; then @@ -105,12 +105,12 @@ if [ ${elapsed} -ge ${TIMEOUT} ]; then exit 1 fi -sleep 2 - #----------------------------------------------------------------------------- # create docs and run test #----------------------------------------------------------------------------- +#set -e + # create documents in client make trigger-create-docs diff --git a/scripts/docker/files/start-client-test.sh b/scripts/docker/files/start-client-test.sh index 5997385f..9dec3371 100755 --- a/scripts/docker/files/start-client-test.sh +++ b/scripts/docker/files/start-client-test.sh @@ -9,8 +9,8 @@ CMD="/usr/local/soledad/test-env.py" REPO="/var/local/soledad" if [ ! -z "${SOLEDAD_REMOTE}" ]; then - git -C ${REPO} remote add test ${SOLEDAD_REMOTE} - git -C ${REPO} fetch test + git -C ${REPO} remote set-url origin ${SOLEDAD_REMOTE} + git -C ${REPO} fetch origin fi if [ ! -z "${SOLEDAD_BRANCH}" ]; then diff --git a/scripts/docker/files/start-server.sh b/scripts/docker/files/start-server.sh index b9b5e4ad..0980d352 100755 --- a/scripts/docker/files/start-server.sh +++ b/scripts/docker/files/start-server.sh @@ -27,8 +27,8 @@ CMD="/usr/local/soledad/test-env.py" REPO="/var/local/soledad" if [ ! -z "${SOLEDAD_REMOTE}" ]; then - git -C ${REPO} remote add test ${SOLEDAD_REMOTE} - git -C ${REPO} fetch test + git -C ${REPO} remote set-url origin ${SOLEDAD_REMOTE} + git -C ${REPO} fetch origin fi if [ ! -z "${SOLEDAD_BRANCH}" ]; then @@ -80,4 +80,5 @@ fi # actually run the server #--------------------------------------------------------------------------- +echo "Starting soledad server..." ${CMD} soledad-server start --no-daemonize diff --git a/scripts/docker/files/start-trial-test.sh b/scripts/docker/files/start-trial-test.sh index 15638b65..ad139288 100755 --- a/scripts/docker/files/start-trial-test.sh +++ b/scripts/docker/files/start-trial-test.sh @@ -9,8 +9,8 @@ CMD="/usr/local/soledad/test-env.py" REPO="/var/local/soledad" if [ ! -z "${SOLEDAD_REMOTE}" ]; then - git -C ${REPO} remote add test ${SOLEDAD_REMOTE} - git -C ${REPO} fetch test + git -C ${REPO} remote set-url origin ${SOLEDAD_REMOTE} + git -C ${REPO} fetch origin fi if [ ! -z "${SOLEDAD_BRANCH}" ]; then -- cgit v1.2.3 From e14f17247ce03f1c49cab3c944039ff9aba84f64 Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 10 Jun 2016 18:07:27 -0300 Subject: [test] improve docker run-test script --- scripts/docker/helper/run-test.sh | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/scripts/docker/helper/run-test.sh b/scripts/docker/helper/run-test.sh index 0a2d6b6b..1b0e3db7 100755 --- a/scripts/docker/helper/run-test.sh +++ b/scripts/docker/helper/run-test.sh @@ -14,14 +14,19 @@ # seconds to wait before giving up waiting from server TIMEOUT=20 -# get a test +# parse command if [ ${#} -ne 1 ]; then - "Usage: ${0} [perf|connect]" + echo "Usage: ${0} perf|connect" exit 1 fi test=${1} +if [ "${1}" != "perf" -a "${1}" != "perf" ]; then + echo "Usage: ${0} perf|connect" + exit 1 +fi + # make sure the image is up to date make image @@ -33,11 +38,9 @@ scriptpath=$(dirname "${script}") tempfile=`mktemp -u` make run-server CONTAINER_ID_FILE=${tempfile} -# get server container info +# wait for server until timeout container_id=`cat ${tempfile}` server_ip=`${scriptpath}/get-container-ip.sh ${container_id}` - -# wait for server until timeout start=`date +%s` elapsed=0 @@ -63,10 +66,6 @@ fi set -e -# run the client -if [ "${test}" = "connect" ]; then - make run-client-test CONTAINER_ID_FILE=${tempfile} -elif [ "${test}" = "perf" ]; then - make run-perf-test CONTAINER_ID_FILE=${tempfile} - make cp-perf-result CONTAINER_ID_FILE=${tempfile} -fi +# run the test +make run-${test}-test CONTAINER_ID_FILE=${tempfile} +rm -r ${tempfile} -- cgit v1.2.3 From 78d61dfaadf9bcac7258a33738c660b238b7bf27 Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 10 Jun 2016 22:07:35 -0300 Subject: [test] refactor of docker scripts --- scripts/docker/Dockerfile | 41 +- scripts/docker/Makefile | 18 +- scripts/docker/files/apt/leap.list | 4 + scripts/docker/files/bin/client_side_db.py | 322 +++++++++++ scripts/docker/files/bin/conf/cert_default.conf | 15 + scripts/docker/files/bin/conf/couchdb_default.ini | 361 ++++++++++++ .../files/bin/conf/soledad-server_default.conf | 5 + scripts/docker/files/bin/run-client-bootstrap.sh | 20 + scripts/docker/files/bin/run-client-perf.sh | 128 +++++ scripts/docker/files/bin/run-server.sh | 89 +++ scripts/docker/files/bin/run-trial.sh | 23 + scripts/docker/files/bin/setup-test-env.py | 640 +++++++++++++++++++++ scripts/docker/files/bin/util.py | 75 +++ scripts/docker/files/bin/util.sh | 12 + .../docker/files/build/install-deps-from-repos.sh | 30 + scripts/docker/files/client_side_db.py | 322 ----------- scripts/docker/files/conf/cert_default.conf | 15 - scripts/docker/files/conf/couchdb_default.ini | 361 ------------ .../docker/files/conf/soledad-server_default.conf | 5 - scripts/docker/files/leap.list | 4 - scripts/docker/files/run-perf-test.sh | 124 ---- scripts/docker/files/setup-env.sh | 55 -- scripts/docker/files/start-client-test.sh | 20 - scripts/docker/files/start-server.sh | 84 --- scripts/docker/files/start-trial-test.sh | 23 - scripts/docker/files/test-env.py | 640 --------------------- scripts/docker/files/util.py | 75 --- scripts/docker/helper/run-test.sh | 8 +- scripts/docker/helper/run-until-error.sh | 12 + 29 files changed, 1769 insertions(+), 1762 deletions(-) create mode 100644 scripts/docker/files/apt/leap.list create mode 100644 scripts/docker/files/bin/client_side_db.py create mode 100644 scripts/docker/files/bin/conf/cert_default.conf create mode 100644 scripts/docker/files/bin/conf/couchdb_default.ini create mode 100644 scripts/docker/files/bin/conf/soledad-server_default.conf create mode 100755 scripts/docker/files/bin/run-client-bootstrap.sh create mode 100755 scripts/docker/files/bin/run-client-perf.sh create mode 100755 scripts/docker/files/bin/run-server.sh create mode 100755 scripts/docker/files/bin/run-trial.sh create mode 100755 scripts/docker/files/bin/setup-test-env.py create mode 100644 scripts/docker/files/bin/util.py create mode 100644 scripts/docker/files/bin/util.sh create mode 100755 scripts/docker/files/build/install-deps-from-repos.sh delete mode 100644 scripts/docker/files/client_side_db.py delete mode 100644 scripts/docker/files/conf/cert_default.conf delete mode 100644 scripts/docker/files/conf/couchdb_default.ini delete mode 100644 scripts/docker/files/conf/soledad-server_default.conf delete mode 100644 scripts/docker/files/leap.list delete mode 100755 scripts/docker/files/run-perf-test.sh delete mode 100755 scripts/docker/files/setup-env.sh delete mode 100755 scripts/docker/files/start-client-test.sh delete mode 100755 scripts/docker/files/start-server.sh delete mode 100755 scripts/docker/files/start-trial-test.sh delete mode 100755 scripts/docker/files/test-env.py delete mode 100644 scripts/docker/files/util.py create mode 100755 scripts/docker/helper/run-until-error.sh diff --git a/scripts/docker/Dockerfile b/scripts/docker/Dockerfile index 36180633..915508ea 100644 --- a/scripts/docker/Dockerfile +++ b/scripts/docker/Dockerfile @@ -5,13 +5,18 @@ FROM debian EXPOSE 2424 # install dependencies from debian repos -COPY files/leap.list /etc/apt/sources.list.d/ +COPY files/apt/leap.list /etc/apt/sources.list.d/ RUN apt-get update RUN apt-get -y --force-yes install leap-archive-keyring RUN apt-get update + RUN apt-get -y install git +RUN apt-get -y install vim +RUN apt-get -y install python-ipdb + +# install python deps RUN apt-get -y install libpython2.7-dev RUN apt-get -y install libffi-dev RUN apt-get -y install libssl-dev @@ -23,30 +28,24 @@ RUN apt-get -y install python-scrypt RUN apt-get -y install leap-keymanager RUN apt-get -y install python-tz -# soledad-perf deps +RUN pip install -U pip +RUN pip install psutil + +# install soledad-perf deps RUN pip install klein -RUN apt-get -y install gnuplot RUN apt-get -y install curl RUN apt-get -y install httperf -# debugging deps -RUN apt-get -y install vim -RUN apt-get -y install python-ipdb +# clone repositories +ENV BASEURL "https://github.com/leapcode" +ENV VARDIR "/var/local" +ENV REPOS "soledad leap_pycommon soledad-perf" +RUN for repo in ${REPOS}; do git clone ${BASEURL}/${repo}.git /var/local/${repo}; done # copy over files to help setup the environment and run soledad RUN mkdir -p /usr/local/soledad -RUN mkdir -p /usr/local/soledad/conf - -# setup the enviroment for running soledad client and server -COPY files/setup-env.sh /usr/local/soledad/ -RUN /usr/local/soledad/setup-env.sh - -# copy runtime files for running server, client, tests, etc on a container -COPY files/client_side_db.py /usr/local/soledad/ -COPY files/start-client-test.sh /usr/local/soledad/ -COPY files/run-perf-test.sh /usr/local/soledad/ -COPY files/start-server.sh /usr/local/soledad/ -COPY files/start-trial-test.sh /usr/local/soledad/ -COPY files/test-env.py /usr/local/soledad/ -COPY files/util.py /usr/local/soledad/ -COPY files/conf/* /usr/local/soledad/conf/ + +COPY files/build/install-deps-from-repos.sh /usr/local/soledad/ +RUN /usr/local/soledad/install-deps-from-repos.sh + +COPY files/bin/ /usr/local/soledad/ diff --git a/scripts/docker/Makefile b/scripts/docker/Makefile index 080fd16c..9dbe9062 100644 --- a/scripts/docker/Makefile +++ b/scripts/docker/Makefile @@ -10,7 +10,7 @@ # Example usage: # # make run-server CONTAINER_ID_FILE=/tmp/container-id.txt -# make run-client-test CONTAINER_ID_FILE=/tmp/container-id.txt +# make run-client-perf CONTAINER_ID_FILE=/tmp/container-id.txt ##################################################################### # Some configurations you might override when calling this makefile # @@ -50,9 +50,9 @@ run-server: --cidfile=$(CONTAINER_ID_FILE) \ --detach \ $(IMAGE_NAME) \ - /usr/local/soledad/start-server.sh + /usr/local/soledad/run-server.sh # --drop-to-shell -run-client-test: +run-client-bootstrap: @if [ -z "$(CONTAINER_ID_FILE)" ]; then \ echo "Error: you have to pass a value to CONTAINER_ID_FILE."; \ exit 2; \ @@ -65,28 +65,28 @@ run-client-test: --env="SOLEDAD_BRANCH=$(SOLEDAD_BRANCH)" \ --env="SOLEDAD_SERVER_URL=http://$${server_ip}:2424" \ $(IMAGE_NAME) \ - /usr/local/soledad/start-client-test.sh + /usr/local/soledad/run-client-bootstrap.sh ################################################# # Run all trial tests inside a docker container # ################################################# -run-trial-test: +run-trial: docker run -t -i \ --memory="$(MEMORY)" \ --env="SOLEDAD_REMOTE=$(SOLEDAD_REMOTE)" \ --env="SOLEDAD_BRANCH=$(SOLEDAD_BRANCH)" \ $(IMAGE_NAME) \ - /usr/local/soledad/start-trial-test.sh + /usr/local/soledad/run-trial.sh ############################################ # Performance tests and graphic generation # ############################################ -run-perf: +run-perf-test: helper/run-test.sh perf -run-perf-test: +run-client-perf: @if [ -z "$(CONTAINER_ID_FILE)" ]; then \ echo "Error: you have to pass a value to CONTAINER_ID_FILE."; \ exit 2; \ @@ -105,7 +105,7 @@ run-perf-test: --env="SOLEDAD_STATS=1" \ --env="SOLEDAD_SERVER_URL=http://$${server_ip}:2424" \ $(IMAGE_NAME) \ - /usr/local/soledad/run-perf-test.sh + /usr/local/soledad/run-client-perf.sh # --drop-to-shell cp-perf-result: @if [ -z "$(CONTAINER_ID_FILE)" ]; then \ diff --git a/scripts/docker/files/apt/leap.list b/scripts/docker/files/apt/leap.list new file mode 100644 index 00000000..7eb474d8 --- /dev/null +++ b/scripts/docker/files/apt/leap.list @@ -0,0 +1,4 @@ +# This file is meant to be copied into the `/etc/apt/sources.list.d` directory +# inside a docker image to provide a source for leap-specific packages. + +deb http://deb.leap.se/0.8 jessie main diff --git a/scripts/docker/files/bin/client_side_db.py b/scripts/docker/files/bin/client_side_db.py new file mode 100644 index 00000000..4be33d13 --- /dev/null +++ b/scripts/docker/files/bin/client_side_db.py @@ -0,0 +1,322 @@ +#!/usr/bin/python + +import os +import argparse +import tempfile +import getpass +import requests +import srp._pysrp as srp +import binascii +import logging +import json +import time + +from twisted.internet import reactor +from twisted.internet.defer import inlineCallbacks + +from leap.soledad.client import Soledad +from leap.keymanager import KeyManager +from leap.keymanager.openpgp import OpenPGPKey + +from leap.common.events import server +server.ensure_server() + +from util import ValidateUserHandle + + +""" +Script to give access to client-side Soledad database. + +This is mainly used for tests, but can also be used to recover data from a +Soledad database (public/private keys, export documents, etc). + +To speed up testing/debugging, this script can dump the auth data after +logging in. Use the --export-auth-data option to export auth data to a file. +The contents of the file is a json dictionary containing the uuid, server_url, +cert_file and token, which is enough info to instantiate a soledad client +without having to interact with the webapp again. Use the --use-auth-data +option to use the auth data stored in a file. + +Use the --help option to see available options. +""" + + +# create a logger +logger = logging.getLogger(__name__) +LOG_FORMAT = '%(asctime)s %(message)s' +logging.basicConfig(format=LOG_FORMAT, level=logging.DEBUG) + + +safe_unhexlify = lambda x: binascii.unhexlify(x) if ( + len(x) % 2 == 0) else binascii.unhexlify('0' + x) + + +def _fail(reason): + logger.error('Fail: ' + reason) + exit(2) + + +def _get_api_info(provider): + info = requests.get( + 'https://' + provider + '/provider.json', verify=False).json() + return info['api_uri'], info['api_version'] + + +def _login(username, passphrase, provider, api_uri, api_version): + usr = srp.User(username, passphrase, srp.SHA256, srp.NG_1024) + auth = None + try: + auth = _authenticate(api_uri, api_version, usr).json() + except requests.exceptions.ConnectionError: + _fail('Could not connect to server.') + if 'errors' in auth: + _fail(str(auth['errors'])) + return api_uri, api_version, auth + + +def _authenticate(api_uri, api_version, usr): + api_url = "%s/%s" % (api_uri, api_version) + session = requests.session() + uname, A = usr.start_authentication() + params = {'login': uname, 'A': binascii.hexlify(A)} + init = session.post( + api_url + '/sessions', data=params, verify=False).json() + if 'errors' in init: + _fail('test user not found') + M = usr.process_challenge( + safe_unhexlify(init['salt']), safe_unhexlify(init['B'])) + return session.put(api_url + '/sessions/' + uname, verify=False, + data={'client_auth': binascii.hexlify(M)}) + + +def _get_soledad_info(username, provider, passphrase, basedir): + api_uri, api_version = _get_api_info(provider) + auth = _login(username, passphrase, provider, api_uri, api_version) + # get soledad server url + service_url = '%s/%s/config/soledad-service.json' % \ + (api_uri, api_version) + soledad_hosts = requests.get(service_url, verify=False).json()['hosts'] + hostnames = soledad_hosts.keys() + # allow for choosing the host + host = hostnames[0] + if len(hostnames) > 1: + i = 1 + print "There are many available hosts:" + for h in hostnames: + print " (%d) %s.%s" % (i, h, provider) + i += 1 + choice = raw_input("Choose a host to use (default: 1): ") + if choice != '': + host = hostnames[int(choice) - 1] + server_url = 'https://%s:%d/user-%s' % \ + (soledad_hosts[host]['hostname'], soledad_hosts[host]['port'], + auth[2]['id']) + # get provider ca certificate + ca_cert = requests.get('https://%s/ca.crt' % provider, verify=False).text + cert_file = os.path.join(basedir, 'ca.crt') + with open(cert_file, 'w') as f: + f.write(ca_cert) + return auth[2]['id'], server_url, cert_file, auth[2]['token'] + + +def _get_soledad_instance(uuid, passphrase, basedir, server_url, cert_file, + token): + # setup soledad info + logger.info('UUID is %s' % uuid) + logger.info('Server URL is %s' % server_url) + secrets_path = os.path.join( + basedir, '%s.secret' % uuid) + local_db_path = os.path.join( + basedir, '%s.db' % uuid) + # instantiate soledad + return Soledad( + uuid, + unicode(passphrase), + secrets_path=secrets_path, + local_db_path=local_db_path, + server_url=server_url, + cert_file=cert_file, + auth_token=token, + defer_encryption=True) + + +def _get_keymanager_instance(username, provider, soledad, token, + ca_cert_path=None, api_uri=None, api_version=None, + uid=None, gpgbinary=None): + return KeyManager( + "{username}@{provider}".format(username=username, provider=provider), + "http://uri", + soledad, + token=token, + ca_cert_path=ca_cert_path, + api_uri=api_uri, + api_version=api_version, + uid=uid, + gpgbinary=gpgbinary) + + +def _parse_args(): + # parse command line + parser = argparse.ArgumentParser() + parser.add_argument( + 'user@provider', action=ValidateUserHandle, help='the user handle') + parser.add_argument( + '--basedir', '-b', default=None, + help='soledad base directory') + parser.add_argument( + '--passphrase', '-p', default=None, + help='the user passphrase') + parser.add_argument( + '--get-all-docs', '-a', action='store_true', + help='get all documents from the local database') + parser.add_argument( + '--create-docs', '-c', default=0, type=int, + help='create a number of documents') + parser.add_argument( + '--sync', '-s', action='store_true', + help='synchronize with the server replica') + parser.add_argument( + '--repeat-sync', '-r', action='store_true', + help='repeat synchronization until no new data is received') + parser.add_argument( + '--export-public-key', help="export the public key to a file") + parser.add_argument( + '--export-private-key', help="export the private key to a file") + parser.add_argument( + '--export-incoming-messages', + help="export incoming messages to a directory") + parser.add_argument( + '--export-auth-data', + help="export authentication data to a file") + parser.add_argument( + '--use-auth-data', + help="use authentication data from a file") + return parser.parse_args() + + +def _get_passphrase(args): + passphrase = args.passphrase + if passphrase is None: + passphrase = getpass.getpass( + 'Password for %s@%s: ' % (args.username, args.provider)) + return passphrase + + +def _get_basedir(args): + basedir = args.basedir + if basedir is None: + basedir = tempfile.mkdtemp() + elif not os.path.isdir(basedir): + os.mkdir(basedir) + logger.info('Using %s as base directory.' % basedir) + return basedir + + +@inlineCallbacks +def _export_key(args, km, fname, private=False): + address = args.username + "@" + args.provider + pkey = yield km.get_key( + address, OpenPGPKey, private=private, fetch_remote=False) + with open(args.export_private_key, "w") as f: + f.write(pkey.key_data) + + +@inlineCallbacks +def _export_incoming_messages(soledad, directory): + yield soledad.create_index("by-incoming", "bool(incoming)") + docs = yield soledad.get_from_index("by-incoming", '1') + i = 1 + for doc in docs: + with open(os.path.join(directory, "message_%d.gpg" % i), "w") as f: + f.write(doc.content["_enc_json"]) + i += 1 + + +@inlineCallbacks +def _get_all_docs(soledad): + _, docs = yield soledad.get_all_docs() + for doc in docs: + print json.dumps(doc.content, indent=4) + + +# main program + +@inlineCallbacks +def _main(soledad, km, args): + try: + if args.create_docs: + for i in xrange(args.create_docs): + t = time.time() + logger.debug( + "Creating doc %d/%d..." % (i + 1, args.create_docs)) + content = { + 'datetime': time.strftime( + "%Y-%m-%d %H:%M:%S", time.gmtime(t)), + 'timestamp': t, + 'index': i, + 'total': args.create_docs, + } + yield soledad.create_doc(content) + if args.sync: + yield soledad.sync() + if args.repeat_sync: + old_gen = 0 + new_gen = yield soledad.sync() + while old_gen != new_gen: + old_gen = new_gen + new_gen = yield soledad.sync() + if args.get_all_docs: + yield _get_all_docs(soledad) + if args.export_private_key: + yield _export_key(args, km, args.export_private_key, private=True) + if args.export_public_key: + yield _export_key(args, km, args.expoert_public_key, private=False) + if args.export_incoming_messages: + yield _export_incoming_messages( + soledad, args.export_incoming_messages) + except Exception as e: + logger.error(e) + finally: + soledad.close() + reactor.callWhenRunning(reactor.stop) + + +if __name__ == '__main__': + args = _parse_args() + passphrase = _get_passphrase(args) + basedir = _get_basedir(args) + + if not args.use_auth_data: + # get auth data from server + uuid, server_url, cert_file, token = \ + _get_soledad_info( + args.username, args.provider, passphrase, basedir) + else: + # load auth data from file + with open(args.use_auth_data) as f: + auth_data = json.loads(f.read()) + uuid = auth_data['uuid'] + server_url = auth_data['server_url'] + cert_file = auth_data['cert_file'] + token = auth_data['token'] + + # export auth data to a file + if args.export_auth_data: + with open(args.export_auth_data, "w") as f: + f.write(json.dumps({ + 'uuid': uuid, + 'server_url': server_url, + 'cert_file': cert_file, + 'token': token, + })) + + soledad = _get_soledad_instance( + uuid, passphrase, basedir, server_url, cert_file, token) + km = _get_keymanager_instance( + args.username, + args.provider, + soledad, + token, + uid=uuid) + _main(soledad, km, args) + reactor.run() diff --git a/scripts/docker/files/bin/conf/cert_default.conf b/scripts/docker/files/bin/conf/cert_default.conf new file mode 100644 index 00000000..8043cea3 --- /dev/null +++ b/scripts/docker/files/bin/conf/cert_default.conf @@ -0,0 +1,15 @@ +[ req ] +default_bits = 1024 +default_keyfile = keyfile.pem +distinguished_name = req_distinguished_name +prompt = no +output_password = mypass + +[ req_distinguished_name ] +C = GB +ST = Test State or Province +L = Test Locality +O = Organization Name +OU = Organizational Unit Name +CN = localhost +emailAddress = test@email.address diff --git a/scripts/docker/files/bin/conf/couchdb_default.ini b/scripts/docker/files/bin/conf/couchdb_default.ini new file mode 100644 index 00000000..5ab72d7b --- /dev/null +++ b/scripts/docker/files/bin/conf/couchdb_default.ini @@ -0,0 +1,361 @@ +; etc/couchdb/default.ini.tpl. Generated from default.ini.tpl.in by configure. + +; Upgrading CouchDB will overwrite this file. +[vendor] +name = The Apache Software Foundation +version = 1.6.0 + +[couchdb] +database_dir = BASEDIR +view_index_dir = BASEDIR +util_driver_dir = /usr/lib/x86_64-linux-gnu/couchdb/erlang/lib/couch-1.6.0/priv/lib +max_document_size = 4294967296 ; 4 GB +os_process_timeout = 5000 ; 5 seconds. for view and external servers. +max_dbs_open = 100 +delayed_commits = true ; set this to false to ensure an fsync before 201 Created is returned +uri_file = BASEDIR/couch.uri +; Method used to compress everything that is appended to database and view index files, except +; for attachments (see the attachments section). Available methods are: +; +; none - no compression +; snappy - use google snappy, a very fast compressor/decompressor +uuid = bc2f8b84ecb0b13a31cf7f6881a52194 + +; deflate_[N] - use zlib's deflate, N is the compression level which ranges from 1 (fastest, +; lowest compression ratio) to 9 (slowest, highest compression ratio) +file_compression = snappy +; Higher values may give better read performance due to less read operations +; and/or more OS page cache hits, but they can also increase overall response +; time for writes when there are many attachment write requests in parallel. +attachment_stream_buffer_size = 4096 + +plugin_dir = /usr/lib/x86_64-linux-gnu/couchdb/plugins + +[database_compaction] +; larger buffer sizes can originate smaller files +doc_buffer_size = 524288 ; value in bytes +checkpoint_after = 5242880 ; checkpoint after every N bytes were written + +[view_compaction] +; larger buffer sizes can originate smaller files +keyvalue_buffer_size = 2097152 ; value in bytes + +[httpd] +port = 5984 +bind_address = 127.0.0.1 +authentication_handlers = {couch_httpd_oauth, oauth_authentication_handler}, {couch_httpd_auth, cookie_authentication_handler}, {couch_httpd_auth, default_authentication_handler} +default_handler = {couch_httpd_db, handle_request} +secure_rewrites = true +vhost_global_handlers = _utils, _uuids, _session, _oauth, _users +allow_jsonp = false +; Options for the MochiWeb HTTP server. +;server_options = [{backlog, 128}, {acceptor_pool_size, 16}] +; For more socket options, consult Erlang's module 'inet' man page. +;socket_options = [{recbuf, 262144}, {sndbuf, 262144}, {nodelay, true}] +socket_options = [{recbuf, 262144}, {sndbuf, 262144}] +log_max_chunk_size = 1000000 +enable_cors = false +; CouchDB can optionally enforce a maximum uri length; +; max_uri_length = 8000 + +[ssl] +port = 6984 + +[log] +file = BASEDIR/couch.log +level = info +include_sasl = true + +[couch_httpd_auth] +authentication_db = _users +authentication_redirect = /_utils/session.html +require_valid_user = false +timeout = 600 ; number of seconds before automatic logout +auth_cache_size = 50 ; size is number of cache entries +allow_persistent_cookies = false ; set to true to allow persistent cookies +iterations = 10 ; iterations for password hashing +; min_iterations = 1 +; max_iterations = 1000000000 +; comma-separated list of public fields, 404 if empty +; public_fields = + +[cors] +credentials = false +; List of origins separated by a comma, * means accept all +; Origins must include the scheme: http://example.com +; You can’t set origins: * and credentials = true at the same time. +;origins = * +; List of accepted headers separated by a comma +; headers = +; List of accepted methods +; methods = + + +; Configuration for a vhost +;[cors:http://example.com] +; credentials = false +; List of origins separated by a comma +; Origins must include the scheme: http://example.com +; You can’t set origins: * and credentials = true at the same time. +;origins = +; List of accepted headers separated by a comma +; headers = +; List of accepted methods +; methods = + +[couch_httpd_oauth] +; If set to 'true', oauth token and consumer secrets will be looked up +; in the authentication database (_users). These secrets are stored in +; a top level property named "oauth" in user documents. Example: +; { +; "_id": "org.couchdb.user:joe", +; "type": "user", +; "name": "joe", +; "password_sha": "fe95df1ca59a9b567bdca5cbaf8412abd6e06121", +; "salt": "4e170ffeb6f34daecfd814dfb4001a73" +; "roles": ["foo", "bar"], +; "oauth": { +; "consumer_keys": { +; "consumerKey1": "key1Secret", +; "consumerKey2": "key2Secret" +; }, +; "tokens": { +; "token1": "token1Secret", +; "token2": "token2Secret" +; } +; } +; } +use_users_db = false + +[query_servers] +javascript = /usr/bin/couchjs /usr/share/couchdb/server/main.js +coffeescript = /usr/bin/couchjs /usr/share/couchdb/server/main-coffee.js + + +; Changing reduce_limit to false will disable reduce_limit. +; If you think you're hitting reduce_limit with a "good" reduce function, +; please let us know on the mailing list so we can fine tune the heuristic. +[query_server_config] +reduce_limit = true +os_process_limit = 25 + +[daemons] +index_server={couch_index_server, start_link, []} +external_manager={couch_external_manager, start_link, []} +query_servers={couch_query_servers, start_link, []} +vhosts={couch_httpd_vhost, start_link, []} +httpd={couch_httpd, start_link, []} +stats_aggregator={couch_stats_aggregator, start, []} +stats_collector={couch_stats_collector, start, []} +uuids={couch_uuids, start, []} +auth_cache={couch_auth_cache, start_link, []} +replicator_manager={couch_replicator_manager, start_link, []} +os_daemons={couch_os_daemons, start_link, []} +compaction_daemon={couch_compaction_daemon, start_link, []} + +[httpd_global_handlers] +/ = {couch_httpd_misc_handlers, handle_welcome_req, <<"Welcome">>} +favicon.ico = {couch_httpd_misc_handlers, handle_favicon_req, "/usr/share/couchdb/www"} + +_utils = {couch_httpd_misc_handlers, handle_utils_dir_req, "/usr/share/couchdb/www"} +_all_dbs = {couch_httpd_misc_handlers, handle_all_dbs_req} +_active_tasks = {couch_httpd_misc_handlers, handle_task_status_req} +_config = {couch_httpd_misc_handlers, handle_config_req} +_replicate = {couch_replicator_httpd, handle_req} +_uuids = {couch_httpd_misc_handlers, handle_uuids_req} +_restart = {couch_httpd_misc_handlers, handle_restart_req} +_stats = {couch_httpd_stats_handlers, handle_stats_req} +_log = {couch_httpd_misc_handlers, handle_log_req} +_session = {couch_httpd_auth, handle_session_req} +_oauth = {couch_httpd_oauth, handle_oauth_req} +_db_updates = {couch_dbupdates_httpd, handle_req} +_plugins = {couch_plugins_httpd, handle_req} + +[httpd_db_handlers] +_all_docs = {couch_mrview_http, handle_all_docs_req} +_changes = {couch_httpd_db, handle_changes_req} +_compact = {couch_httpd_db, handle_compact_req} +_design = {couch_httpd_db, handle_design_req} +_temp_view = {couch_mrview_http, handle_temp_view_req} +_view_cleanup = {couch_mrview_http, handle_cleanup_req} + +; The external module takes an optional argument allowing you to narrow it to a +; single script. Otherwise the script name is inferred from the first path section +; after _external's own path. +; _mypath = {couch_httpd_external, handle_external_req, <<"mykey">>} +; _external = {couch_httpd_external, handle_external_req} + +[httpd_design_handlers] +_compact = {couch_mrview_http, handle_compact_req} +_info = {couch_mrview_http, handle_info_req} +_list = {couch_mrview_show, handle_view_list_req} +_rewrite = {couch_httpd_rewrite, handle_rewrite_req} +_show = {couch_mrview_show, handle_doc_show_req} +_update = {couch_mrview_show, handle_doc_update_req} +_view = {couch_mrview_http, handle_view_req} + +; enable external as an httpd handler, then link it with commands here. +; note, this api is still under consideration. +; [external] +; mykey = /path/to/mycommand + +; Here you can setup commands for CouchDB to manage +; while it is alive. It will attempt to keep each command +; alive if it exits. +; [os_daemons] +; some_daemon_name = /path/to/script -with args + + +[uuids] +; Known algorithms: +; random - 128 bits of random awesome +; All awesome, all the time. +; sequential - monotonically increasing ids with random increments +; First 26 hex characters are random. Last 6 increment in +; random amounts until an overflow occurs. On overflow, the +; random prefix is regenerated and the process starts over. +; utc_random - Time since Jan 1, 1970 UTC with microseconds +; First 14 characters are the time in hex. Last 18 are random. +; utc_id - Time since Jan 1, 1970 UTC with microseconds, plus utc_id_suffix string +; First 14 characters are the time in hex. uuids/utc_id_suffix string value is appended to these. +algorithm = sequential +; The utc_id_suffix value will be appended to uuids generated by the utc_id algorithm. +; Replicating instances should have unique utc_id_suffix values to ensure uniqueness of utc_id ids. +utc_id_suffix = +# Maximum number of UUIDs retrievable from /_uuids in a single request +max_count = 1000 + +[stats] +; rate is in milliseconds +rate = 1000 +; sample intervals are in seconds +samples = [0, 60, 300, 900] + +[attachments] +compression_level = 8 ; from 1 (lowest, fastest) to 9 (highest, slowest), 0 to disable compression +compressible_types = text/*, application/javascript, application/json, application/xml + +[replicator] +db = _replicator +; Maximum replicaton retry count can be a non-negative integer or "infinity". +max_replication_retry_count = 10 +; More worker processes can give higher network throughput but can also +; imply more disk and network IO. +worker_processes = 4 +; With lower batch sizes checkpoints are done more frequently. Lower batch sizes +; also reduce the total amount of used RAM memory. +worker_batch_size = 500 +; Maximum number of HTTP connections per replication. +http_connections = 20 +; HTTP connection timeout per replication. +; Even for very fast/reliable networks it might need to be increased if a remote +; database is too busy. +connection_timeout = 30000 +; If a request fails, the replicator will retry it up to N times. +retries_per_request = 10 +; Some socket options that might boost performance in some scenarios: +; {nodelay, boolean()} +; {sndbuf, integer()} +; {recbuf, integer()} +; {priority, integer()} +; See the `inet` Erlang module's man page for the full list of options. +socket_options = [{keepalive, true}, {nodelay, false}] +; Path to a file containing the user's certificate. +;cert_file = /full/path/to/server_cert.pem +; Path to file containing user's private PEM encoded key. +;key_file = /full/path/to/server_key.pem +; String containing the user's password. Only used if the private keyfile is password protected. +;password = somepassword +; Set to true to validate peer certificates. +verify_ssl_certificates = false +; File containing a list of peer trusted certificates (in the PEM format). +;ssl_trusted_certificates_file = /etc/ssl/certs/ca-certificates.crt +; Maximum peer certificate depth (must be set even if certificate validation is off). +ssl_certificate_max_depth = 3 + +[compaction_daemon] +; The delay, in seconds, between each check for which database and view indexes +; need to be compacted. +check_interval = 300 +; If a database or view index file is smaller then this value (in bytes), +; compaction will not happen. Very small files always have a very high +; fragmentation therefore it's not worth to compact them. +min_file_size = 131072 + +[compactions] +; List of compaction rules for the compaction daemon. +; The daemon compacts databases and their respective view groups when all the +; condition parameters are satisfied. Configuration can be per database or +; global, and it has the following format: +; +; database_name = [ {ParamName, ParamValue}, {ParamName, ParamValue}, ... ] +; _default = [ {ParamName, ParamValue}, {ParamName, ParamValue}, ... ] +; +; Possible parameters: +; +; * db_fragmentation - If the ratio (as an integer percentage), of the amount +; of old data (and its supporting metadata) over the database +; file size is equal to or greater then this value, this +; database compaction condition is satisfied. +; This value is computed as: +; +; (file_size - data_size) / file_size * 100 +; +; The data_size and file_size values can be obtained when +; querying a database's information URI (GET /dbname/). +; +; * view_fragmentation - If the ratio (as an integer percentage), of the amount +; of old data (and its supporting metadata) over the view +; index (view group) file size is equal to or greater then +; this value, then this view index compaction condition is +; satisfied. This value is computed as: +; +; (file_size - data_size) / file_size * 100 +; +; The data_size and file_size values can be obtained when +; querying a view group's information URI +; (GET /dbname/_design/groupname/_info). +; +; * from _and_ to - The period for which a database (and its view groups) compaction +; is allowed. The value for these parameters must obey the format: +; +; HH:MM - HH:MM (HH in [0..23], MM in [0..59]) +; +; * strict_window - If a compaction is still running after the end of the allowed +; period, it will be canceled if this parameter is set to 'true'. +; It defaults to 'false' and it's meaningful only if the *period* +; parameter is also specified. +; +; * parallel_view_compaction - If set to 'true', the database and its views are +; compacted in parallel. This is only useful on +; certain setups, like for example when the database +; and view index directories point to different +; disks. It defaults to 'false'. +; +; Before a compaction is triggered, an estimation of how much free disk space is +; needed is computed. This estimation corresponds to 2 times the data size of +; the database or view index. When there's not enough free disk space to compact +; a particular database or view index, a warning message is logged. +; +; Examples: +; +; 1) [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}] +; The `foo` database is compacted if its fragmentation is 70% or more. +; Any view index of this database is compacted only if its fragmentation +; is 60% or more. +; +; 2) [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "00:00"}, {to, "04:00"}] +; Similar to the preceding example but a compaction (database or view index) +; is only triggered if the current time is between midnight and 4 AM. +; +; 3) [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "00:00"}, {to, "04:00"}, {strict_window, true}] +; Similar to the preceding example - a compaction (database or view index) +; is only triggered if the current time is between midnight and 4 AM. If at +; 4 AM the database or one of its views is still compacting, the compaction +; process will be canceled. +; +; 4) [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "00:00"}, {to, "04:00"}, {strict_window, true}, {parallel_view_compaction, true}] +; Similar to the preceding example, but a database and its views can be +; compacted in parallel. +; +;_default = [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "23:00"}, {to, "04:00"}] diff --git a/scripts/docker/files/bin/conf/soledad-server_default.conf b/scripts/docker/files/bin/conf/soledad-server_default.conf new file mode 100644 index 00000000..5e286374 --- /dev/null +++ b/scripts/docker/files/bin/conf/soledad-server_default.conf @@ -0,0 +1,5 @@ +[soledad-server] +couch_url = http://localhost:5984 +create_cmd = sudo -u soledad-admin /usr/bin/create-user-db +admin_netrc = /etc/couchdb/couchdb-soledad-admin.netrc +batching = 0 diff --git a/scripts/docker/files/bin/run-client-bootstrap.sh b/scripts/docker/files/bin/run-client-bootstrap.sh new file mode 100755 index 00000000..fbbb42e8 --- /dev/null +++ b/scripts/docker/files/bin/run-client-bootstrap.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Run a Soledad client connection test. +# +# This script is meant to be copied to the docker container and run upon +# container start. + +CMD="/usr/local/soledad/setup-test-env.py" +REPO="/var/local/soledad" + +if [ ! -z "${SOLEDAD_REMOTE}" ]; then + git -C ${REPO} remote set-url origin ${SOLEDAD_REMOTE} + git -C ${REPO} fetch origin +fi + +if [ ! -z "${SOLEDAD_BRANCH}" ]; then + git -C ${REPO} checkout ${SOLEDAD_BRANCH} +fi + +${CMD} soledad-client test --server-url ${SOLEDAD_SERVER_URL} diff --git a/scripts/docker/files/bin/run-client-perf.sh b/scripts/docker/files/bin/run-client-perf.sh new file mode 100755 index 00000000..01b27b98 --- /dev/null +++ b/scripts/docker/files/bin/run-client-perf.sh @@ -0,0 +1,128 @@ +#!/bin/sh + +# Start a soledad-perf test using a remote server. +# +# The script does the following: +# +# - configure a remote repository for soledad repo if SOLEDAD_REMOTE is set. +# +# - checkout a specific branch if SOLEDAD_BRANCH is set. +# +# - run the soledad-perf local twisted server that runs the client. Note +# that the actual soledad server should be running on another docker +# container. This local server is only used to measure responsiveness of +# soledad client. The script waits for the server to come up before +# continuing, or else times out after TIMEOUT seconds. +# +# - trigger the creation of documents for sync. +# +# - start the measurement of server responsiveness and sync stages. +# +# - stop the test. +# +# This script is meant to be copied to the docker container and run upon +# container start. + +CMD="/usr/local/soledad/setup-test-env.py" +REPO="/var/local/soledad" +TIMEOUT=20 + +#----------------------------------------------------------------------------- +# configure a remote and checkout a branch +#----------------------------------------------------------------------------- + +if [ ! -z "${SOLEDAD_REMOTE}" ]; then + git -C ${REPO} remote set-url origin ${SOLEDAD_REMOTE} + git -C ${REPO} fetch origin +fi + +if [ ! -z "${SOLEDAD_BRANCH}" ]; then + git -C ${REPO} checkout ${SOLEDAD_BRANCH} +fi + +if [ ! -z "${SOLEDAD_PERF_REMOTE}" ]; then + git -C /var/local/soledad-perf remote set-url origin ${SOLEDAD_PERF_REMOTE} + git -C /var/local/soledad-perf fetch origin +fi + +if [ ! -z "${SOLEDAD_PERF_BRANCH}" ]; then + git -C /var/local/soledad-perf checkout ${SOLEDAD_PERF_BRANCH} +fi + +#----------------------------------------------------------------------------- +# write a configuration file for the perf test +#----------------------------------------------------------------------------- + +cd /var/local/soledad-perf + +cat > defaults.conf < /dev/null & +sleep 5 # wait a bit for some data points + +# run a sync and generate a graph +make trigger-sync +make trigger-stop diff --git a/scripts/docker/files/bin/run-server.sh b/scripts/docker/files/bin/run-server.sh new file mode 100755 index 00000000..feedee7e --- /dev/null +++ b/scripts/docker/files/bin/run-server.sh @@ -0,0 +1,89 @@ +#!/bin/sh + +# Start a soledad server inside a docker container. +# +# This script will: +# +# - eventually checkout a specific branch from a specific soledad remote. +# +# - create everything a soledad server needs to run (certificate, backend +# server database, tables, etc. +# +# - eventually preload the server database with a number of documents equal +# to SOLEDAD_PRELOAD_NUM, and with payload size equal to +# SOLEDAD_PRELOAD_SIZE. +# +# - run the soledad server. +# +# This script is meant to be copied to the docker container and run upon +# container start. + +CMD="/usr/local/soledad/setup-test-env.py" + +#--------------------------------------------------------------------------- +# eventually checkout a specific branch from a specific remote +#--------------------------------------------------------------------------- + +REPO="/var/local/soledad" + +if [ ! -z "${SOLEDAD_REMOTE}" ]; then + git -C ${REPO} remote set-url origin ${SOLEDAD_REMOTE} + git -C ${REPO} fetch origin +fi + +if [ ! -z "${SOLEDAD_BRANCH}" ]; then + git -C ${REPO} checkout ${SOLEDAD_BRANCH} +fi + +#--------------------------------------------------------------------------- +# setup environment for running soledad server +#--------------------------------------------------------------------------- + +${CMD} couch start +${CMD} user-db create +${CMD} token-db create +${CMD} token-db insert-token +${CMD} shared-db create +${CMD} cert create + +#--------------------------------------------------------------------------- +# write a configuration file for the perf test +#--------------------------------------------------------------------------- + +if [ "${SOLEDAD_PRELOAD_NUM}" -gt 0 ]; then + cd /var/local/soledad-perf + + cat > defaults.conf < 1: - i = 1 - print "There are many available hosts:" - for h in hostnames: - print " (%d) %s.%s" % (i, h, provider) - i += 1 - choice = raw_input("Choose a host to use (default: 1): ") - if choice != '': - host = hostnames[int(choice) - 1] - server_url = 'https://%s:%d/user-%s' % \ - (soledad_hosts[host]['hostname'], soledad_hosts[host]['port'], - auth[2]['id']) - # get provider ca certificate - ca_cert = requests.get('https://%s/ca.crt' % provider, verify=False).text - cert_file = os.path.join(basedir, 'ca.crt') - with open(cert_file, 'w') as f: - f.write(ca_cert) - return auth[2]['id'], server_url, cert_file, auth[2]['token'] - - -def _get_soledad_instance(uuid, passphrase, basedir, server_url, cert_file, - token): - # setup soledad info - logger.info('UUID is %s' % uuid) - logger.info('Server URL is %s' % server_url) - secrets_path = os.path.join( - basedir, '%s.secret' % uuid) - local_db_path = os.path.join( - basedir, '%s.db' % uuid) - # instantiate soledad - return Soledad( - uuid, - unicode(passphrase), - secrets_path=secrets_path, - local_db_path=local_db_path, - server_url=server_url, - cert_file=cert_file, - auth_token=token, - defer_encryption=True) - - -def _get_keymanager_instance(username, provider, soledad, token, - ca_cert_path=None, api_uri=None, api_version=None, - uid=None, gpgbinary=None): - return KeyManager( - "{username}@{provider}".format(username=username, provider=provider), - "http://uri", - soledad, - token=token, - ca_cert_path=ca_cert_path, - api_uri=api_uri, - api_version=api_version, - uid=uid, - gpgbinary=gpgbinary) - - -def _parse_args(): - # parse command line - parser = argparse.ArgumentParser() - parser.add_argument( - 'user@provider', action=ValidateUserHandle, help='the user handle') - parser.add_argument( - '--basedir', '-b', default=None, - help='soledad base directory') - parser.add_argument( - '--passphrase', '-p', default=None, - help='the user passphrase') - parser.add_argument( - '--get-all-docs', '-a', action='store_true', - help='get all documents from the local database') - parser.add_argument( - '--create-docs', '-c', default=0, type=int, - help='create a number of documents') - parser.add_argument( - '--sync', '-s', action='store_true', - help='synchronize with the server replica') - parser.add_argument( - '--repeat-sync', '-r', action='store_true', - help='repeat synchronization until no new data is received') - parser.add_argument( - '--export-public-key', help="export the public key to a file") - parser.add_argument( - '--export-private-key', help="export the private key to a file") - parser.add_argument( - '--export-incoming-messages', - help="export incoming messages to a directory") - parser.add_argument( - '--export-auth-data', - help="export authentication data to a file") - parser.add_argument( - '--use-auth-data', - help="use authentication data from a file") - return parser.parse_args() - - -def _get_passphrase(args): - passphrase = args.passphrase - if passphrase is None: - passphrase = getpass.getpass( - 'Password for %s@%s: ' % (args.username, args.provider)) - return passphrase - - -def _get_basedir(args): - basedir = args.basedir - if basedir is None: - basedir = tempfile.mkdtemp() - elif not os.path.isdir(basedir): - os.mkdir(basedir) - logger.info('Using %s as base directory.' % basedir) - return basedir - - -@inlineCallbacks -def _export_key(args, km, fname, private=False): - address = args.username + "@" + args.provider - pkey = yield km.get_key( - address, OpenPGPKey, private=private, fetch_remote=False) - with open(args.export_private_key, "w") as f: - f.write(pkey.key_data) - - -@inlineCallbacks -def _export_incoming_messages(soledad, directory): - yield soledad.create_index("by-incoming", "bool(incoming)") - docs = yield soledad.get_from_index("by-incoming", '1') - i = 1 - for doc in docs: - with open(os.path.join(directory, "message_%d.gpg" % i), "w") as f: - f.write(doc.content["_enc_json"]) - i += 1 - - -@inlineCallbacks -def _get_all_docs(soledad): - _, docs = yield soledad.get_all_docs() - for doc in docs: - print json.dumps(doc.content, indent=4) - - -# main program - -@inlineCallbacks -def _main(soledad, km, args): - try: - if args.create_docs: - for i in xrange(args.create_docs): - t = time.time() - logger.debug( - "Creating doc %d/%d..." % (i + 1, args.create_docs)) - content = { - 'datetime': time.strftime( - "%Y-%m-%d %H:%M:%S", time.gmtime(t)), - 'timestamp': t, - 'index': i, - 'total': args.create_docs, - } - yield soledad.create_doc(content) - if args.sync: - yield soledad.sync() - if args.repeat_sync: - old_gen = 0 - new_gen = yield soledad.sync() - while old_gen != new_gen: - old_gen = new_gen - new_gen = yield soledad.sync() - if args.get_all_docs: - yield _get_all_docs(soledad) - if args.export_private_key: - yield _export_key(args, km, args.export_private_key, private=True) - if args.export_public_key: - yield _export_key(args, km, args.expoert_public_key, private=False) - if args.export_incoming_messages: - yield _export_incoming_messages( - soledad, args.export_incoming_messages) - except Exception as e: - logger.error(e) - finally: - soledad.close() - reactor.callWhenRunning(reactor.stop) - - -if __name__ == '__main__': - args = _parse_args() - passphrase = _get_passphrase(args) - basedir = _get_basedir(args) - - if not args.use_auth_data: - # get auth data from server - uuid, server_url, cert_file, token = \ - _get_soledad_info( - args.username, args.provider, passphrase, basedir) - else: - # load auth data from file - with open(args.use_auth_data) as f: - auth_data = json.loads(f.read()) - uuid = auth_data['uuid'] - server_url = auth_data['server_url'] - cert_file = auth_data['cert_file'] - token = auth_data['token'] - - # export auth data to a file - if args.export_auth_data: - with open(args.export_auth_data, "w") as f: - f.write(json.dumps({ - 'uuid': uuid, - 'server_url': server_url, - 'cert_file': cert_file, - 'token': token, - })) - - soledad = _get_soledad_instance( - uuid, passphrase, basedir, server_url, cert_file, token) - km = _get_keymanager_instance( - args.username, - args.provider, - soledad, - token, - uid=uuid) - _main(soledad, km, args) - reactor.run() diff --git a/scripts/docker/files/conf/cert_default.conf b/scripts/docker/files/conf/cert_default.conf deleted file mode 100644 index 8043cea3..00000000 --- a/scripts/docker/files/conf/cert_default.conf +++ /dev/null @@ -1,15 +0,0 @@ -[ req ] -default_bits = 1024 -default_keyfile = keyfile.pem -distinguished_name = req_distinguished_name -prompt = no -output_password = mypass - -[ req_distinguished_name ] -C = GB -ST = Test State or Province -L = Test Locality -O = Organization Name -OU = Organizational Unit Name -CN = localhost -emailAddress = test@email.address diff --git a/scripts/docker/files/conf/couchdb_default.ini b/scripts/docker/files/conf/couchdb_default.ini deleted file mode 100644 index 5ab72d7b..00000000 --- a/scripts/docker/files/conf/couchdb_default.ini +++ /dev/null @@ -1,361 +0,0 @@ -; etc/couchdb/default.ini.tpl. Generated from default.ini.tpl.in by configure. - -; Upgrading CouchDB will overwrite this file. -[vendor] -name = The Apache Software Foundation -version = 1.6.0 - -[couchdb] -database_dir = BASEDIR -view_index_dir = BASEDIR -util_driver_dir = /usr/lib/x86_64-linux-gnu/couchdb/erlang/lib/couch-1.6.0/priv/lib -max_document_size = 4294967296 ; 4 GB -os_process_timeout = 5000 ; 5 seconds. for view and external servers. -max_dbs_open = 100 -delayed_commits = true ; set this to false to ensure an fsync before 201 Created is returned -uri_file = BASEDIR/couch.uri -; Method used to compress everything that is appended to database and view index files, except -; for attachments (see the attachments section). Available methods are: -; -; none - no compression -; snappy - use google snappy, a very fast compressor/decompressor -uuid = bc2f8b84ecb0b13a31cf7f6881a52194 - -; deflate_[N] - use zlib's deflate, N is the compression level which ranges from 1 (fastest, -; lowest compression ratio) to 9 (slowest, highest compression ratio) -file_compression = snappy -; Higher values may give better read performance due to less read operations -; and/or more OS page cache hits, but they can also increase overall response -; time for writes when there are many attachment write requests in parallel. -attachment_stream_buffer_size = 4096 - -plugin_dir = /usr/lib/x86_64-linux-gnu/couchdb/plugins - -[database_compaction] -; larger buffer sizes can originate smaller files -doc_buffer_size = 524288 ; value in bytes -checkpoint_after = 5242880 ; checkpoint after every N bytes were written - -[view_compaction] -; larger buffer sizes can originate smaller files -keyvalue_buffer_size = 2097152 ; value in bytes - -[httpd] -port = 5984 -bind_address = 127.0.0.1 -authentication_handlers = {couch_httpd_oauth, oauth_authentication_handler}, {couch_httpd_auth, cookie_authentication_handler}, {couch_httpd_auth, default_authentication_handler} -default_handler = {couch_httpd_db, handle_request} -secure_rewrites = true -vhost_global_handlers = _utils, _uuids, _session, _oauth, _users -allow_jsonp = false -; Options for the MochiWeb HTTP server. -;server_options = [{backlog, 128}, {acceptor_pool_size, 16}] -; For more socket options, consult Erlang's module 'inet' man page. -;socket_options = [{recbuf, 262144}, {sndbuf, 262144}, {nodelay, true}] -socket_options = [{recbuf, 262144}, {sndbuf, 262144}] -log_max_chunk_size = 1000000 -enable_cors = false -; CouchDB can optionally enforce a maximum uri length; -; max_uri_length = 8000 - -[ssl] -port = 6984 - -[log] -file = BASEDIR/couch.log -level = info -include_sasl = true - -[couch_httpd_auth] -authentication_db = _users -authentication_redirect = /_utils/session.html -require_valid_user = false -timeout = 600 ; number of seconds before automatic logout -auth_cache_size = 50 ; size is number of cache entries -allow_persistent_cookies = false ; set to true to allow persistent cookies -iterations = 10 ; iterations for password hashing -; min_iterations = 1 -; max_iterations = 1000000000 -; comma-separated list of public fields, 404 if empty -; public_fields = - -[cors] -credentials = false -; List of origins separated by a comma, * means accept all -; Origins must include the scheme: http://example.com -; You can’t set origins: * and credentials = true at the same time. -;origins = * -; List of accepted headers separated by a comma -; headers = -; List of accepted methods -; methods = - - -; Configuration for a vhost -;[cors:http://example.com] -; credentials = false -; List of origins separated by a comma -; Origins must include the scheme: http://example.com -; You can’t set origins: * and credentials = true at the same time. -;origins = -; List of accepted headers separated by a comma -; headers = -; List of accepted methods -; methods = - -[couch_httpd_oauth] -; If set to 'true', oauth token and consumer secrets will be looked up -; in the authentication database (_users). These secrets are stored in -; a top level property named "oauth" in user documents. Example: -; { -; "_id": "org.couchdb.user:joe", -; "type": "user", -; "name": "joe", -; "password_sha": "fe95df1ca59a9b567bdca5cbaf8412abd6e06121", -; "salt": "4e170ffeb6f34daecfd814dfb4001a73" -; "roles": ["foo", "bar"], -; "oauth": { -; "consumer_keys": { -; "consumerKey1": "key1Secret", -; "consumerKey2": "key2Secret" -; }, -; "tokens": { -; "token1": "token1Secret", -; "token2": "token2Secret" -; } -; } -; } -use_users_db = false - -[query_servers] -javascript = /usr/bin/couchjs /usr/share/couchdb/server/main.js -coffeescript = /usr/bin/couchjs /usr/share/couchdb/server/main-coffee.js - - -; Changing reduce_limit to false will disable reduce_limit. -; If you think you're hitting reduce_limit with a "good" reduce function, -; please let us know on the mailing list so we can fine tune the heuristic. -[query_server_config] -reduce_limit = true -os_process_limit = 25 - -[daemons] -index_server={couch_index_server, start_link, []} -external_manager={couch_external_manager, start_link, []} -query_servers={couch_query_servers, start_link, []} -vhosts={couch_httpd_vhost, start_link, []} -httpd={couch_httpd, start_link, []} -stats_aggregator={couch_stats_aggregator, start, []} -stats_collector={couch_stats_collector, start, []} -uuids={couch_uuids, start, []} -auth_cache={couch_auth_cache, start_link, []} -replicator_manager={couch_replicator_manager, start_link, []} -os_daemons={couch_os_daemons, start_link, []} -compaction_daemon={couch_compaction_daemon, start_link, []} - -[httpd_global_handlers] -/ = {couch_httpd_misc_handlers, handle_welcome_req, <<"Welcome">>} -favicon.ico = {couch_httpd_misc_handlers, handle_favicon_req, "/usr/share/couchdb/www"} - -_utils = {couch_httpd_misc_handlers, handle_utils_dir_req, "/usr/share/couchdb/www"} -_all_dbs = {couch_httpd_misc_handlers, handle_all_dbs_req} -_active_tasks = {couch_httpd_misc_handlers, handle_task_status_req} -_config = {couch_httpd_misc_handlers, handle_config_req} -_replicate = {couch_replicator_httpd, handle_req} -_uuids = {couch_httpd_misc_handlers, handle_uuids_req} -_restart = {couch_httpd_misc_handlers, handle_restart_req} -_stats = {couch_httpd_stats_handlers, handle_stats_req} -_log = {couch_httpd_misc_handlers, handle_log_req} -_session = {couch_httpd_auth, handle_session_req} -_oauth = {couch_httpd_oauth, handle_oauth_req} -_db_updates = {couch_dbupdates_httpd, handle_req} -_plugins = {couch_plugins_httpd, handle_req} - -[httpd_db_handlers] -_all_docs = {couch_mrview_http, handle_all_docs_req} -_changes = {couch_httpd_db, handle_changes_req} -_compact = {couch_httpd_db, handle_compact_req} -_design = {couch_httpd_db, handle_design_req} -_temp_view = {couch_mrview_http, handle_temp_view_req} -_view_cleanup = {couch_mrview_http, handle_cleanup_req} - -; The external module takes an optional argument allowing you to narrow it to a -; single script. Otherwise the script name is inferred from the first path section -; after _external's own path. -; _mypath = {couch_httpd_external, handle_external_req, <<"mykey">>} -; _external = {couch_httpd_external, handle_external_req} - -[httpd_design_handlers] -_compact = {couch_mrview_http, handle_compact_req} -_info = {couch_mrview_http, handle_info_req} -_list = {couch_mrview_show, handle_view_list_req} -_rewrite = {couch_httpd_rewrite, handle_rewrite_req} -_show = {couch_mrview_show, handle_doc_show_req} -_update = {couch_mrview_show, handle_doc_update_req} -_view = {couch_mrview_http, handle_view_req} - -; enable external as an httpd handler, then link it with commands here. -; note, this api is still under consideration. -; [external] -; mykey = /path/to/mycommand - -; Here you can setup commands for CouchDB to manage -; while it is alive. It will attempt to keep each command -; alive if it exits. -; [os_daemons] -; some_daemon_name = /path/to/script -with args - - -[uuids] -; Known algorithms: -; random - 128 bits of random awesome -; All awesome, all the time. -; sequential - monotonically increasing ids with random increments -; First 26 hex characters are random. Last 6 increment in -; random amounts until an overflow occurs. On overflow, the -; random prefix is regenerated and the process starts over. -; utc_random - Time since Jan 1, 1970 UTC with microseconds -; First 14 characters are the time in hex. Last 18 are random. -; utc_id - Time since Jan 1, 1970 UTC with microseconds, plus utc_id_suffix string -; First 14 characters are the time in hex. uuids/utc_id_suffix string value is appended to these. -algorithm = sequential -; The utc_id_suffix value will be appended to uuids generated by the utc_id algorithm. -; Replicating instances should have unique utc_id_suffix values to ensure uniqueness of utc_id ids. -utc_id_suffix = -# Maximum number of UUIDs retrievable from /_uuids in a single request -max_count = 1000 - -[stats] -; rate is in milliseconds -rate = 1000 -; sample intervals are in seconds -samples = [0, 60, 300, 900] - -[attachments] -compression_level = 8 ; from 1 (lowest, fastest) to 9 (highest, slowest), 0 to disable compression -compressible_types = text/*, application/javascript, application/json, application/xml - -[replicator] -db = _replicator -; Maximum replicaton retry count can be a non-negative integer or "infinity". -max_replication_retry_count = 10 -; More worker processes can give higher network throughput but can also -; imply more disk and network IO. -worker_processes = 4 -; With lower batch sizes checkpoints are done more frequently. Lower batch sizes -; also reduce the total amount of used RAM memory. -worker_batch_size = 500 -; Maximum number of HTTP connections per replication. -http_connections = 20 -; HTTP connection timeout per replication. -; Even for very fast/reliable networks it might need to be increased if a remote -; database is too busy. -connection_timeout = 30000 -; If a request fails, the replicator will retry it up to N times. -retries_per_request = 10 -; Some socket options that might boost performance in some scenarios: -; {nodelay, boolean()} -; {sndbuf, integer()} -; {recbuf, integer()} -; {priority, integer()} -; See the `inet` Erlang module's man page for the full list of options. -socket_options = [{keepalive, true}, {nodelay, false}] -; Path to a file containing the user's certificate. -;cert_file = /full/path/to/server_cert.pem -; Path to file containing user's private PEM encoded key. -;key_file = /full/path/to/server_key.pem -; String containing the user's password. Only used if the private keyfile is password protected. -;password = somepassword -; Set to true to validate peer certificates. -verify_ssl_certificates = false -; File containing a list of peer trusted certificates (in the PEM format). -;ssl_trusted_certificates_file = /etc/ssl/certs/ca-certificates.crt -; Maximum peer certificate depth (must be set even if certificate validation is off). -ssl_certificate_max_depth = 3 - -[compaction_daemon] -; The delay, in seconds, between each check for which database and view indexes -; need to be compacted. -check_interval = 300 -; If a database or view index file is smaller then this value (in bytes), -; compaction will not happen. Very small files always have a very high -; fragmentation therefore it's not worth to compact them. -min_file_size = 131072 - -[compactions] -; List of compaction rules for the compaction daemon. -; The daemon compacts databases and their respective view groups when all the -; condition parameters are satisfied. Configuration can be per database or -; global, and it has the following format: -; -; database_name = [ {ParamName, ParamValue}, {ParamName, ParamValue}, ... ] -; _default = [ {ParamName, ParamValue}, {ParamName, ParamValue}, ... ] -; -; Possible parameters: -; -; * db_fragmentation - If the ratio (as an integer percentage), of the amount -; of old data (and its supporting metadata) over the database -; file size is equal to or greater then this value, this -; database compaction condition is satisfied. -; This value is computed as: -; -; (file_size - data_size) / file_size * 100 -; -; The data_size and file_size values can be obtained when -; querying a database's information URI (GET /dbname/). -; -; * view_fragmentation - If the ratio (as an integer percentage), of the amount -; of old data (and its supporting metadata) over the view -; index (view group) file size is equal to or greater then -; this value, then this view index compaction condition is -; satisfied. This value is computed as: -; -; (file_size - data_size) / file_size * 100 -; -; The data_size and file_size values can be obtained when -; querying a view group's information URI -; (GET /dbname/_design/groupname/_info). -; -; * from _and_ to - The period for which a database (and its view groups) compaction -; is allowed. The value for these parameters must obey the format: -; -; HH:MM - HH:MM (HH in [0..23], MM in [0..59]) -; -; * strict_window - If a compaction is still running after the end of the allowed -; period, it will be canceled if this parameter is set to 'true'. -; It defaults to 'false' and it's meaningful only if the *period* -; parameter is also specified. -; -; * parallel_view_compaction - If set to 'true', the database and its views are -; compacted in parallel. This is only useful on -; certain setups, like for example when the database -; and view index directories point to different -; disks. It defaults to 'false'. -; -; Before a compaction is triggered, an estimation of how much free disk space is -; needed is computed. This estimation corresponds to 2 times the data size of -; the database or view index. When there's not enough free disk space to compact -; a particular database or view index, a warning message is logged. -; -; Examples: -; -; 1) [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}] -; The `foo` database is compacted if its fragmentation is 70% or more. -; Any view index of this database is compacted only if its fragmentation -; is 60% or more. -; -; 2) [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "00:00"}, {to, "04:00"}] -; Similar to the preceding example but a compaction (database or view index) -; is only triggered if the current time is between midnight and 4 AM. -; -; 3) [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "00:00"}, {to, "04:00"}, {strict_window, true}] -; Similar to the preceding example - a compaction (database or view index) -; is only triggered if the current time is between midnight and 4 AM. If at -; 4 AM the database or one of its views is still compacting, the compaction -; process will be canceled. -; -; 4) [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "00:00"}, {to, "04:00"}, {strict_window, true}, {parallel_view_compaction, true}] -; Similar to the preceding example, but a database and its views can be -; compacted in parallel. -; -;_default = [{db_fragmentation, "70%"}, {view_fragmentation, "60%"}, {from, "23:00"}, {to, "04:00"}] diff --git a/scripts/docker/files/conf/soledad-server_default.conf b/scripts/docker/files/conf/soledad-server_default.conf deleted file mode 100644 index 5e286374..00000000 --- a/scripts/docker/files/conf/soledad-server_default.conf +++ /dev/null @@ -1,5 +0,0 @@ -[soledad-server] -couch_url = http://localhost:5984 -create_cmd = sudo -u soledad-admin /usr/bin/create-user-db -admin_netrc = /etc/couchdb/couchdb-soledad-admin.netrc -batching = 0 diff --git a/scripts/docker/files/leap.list b/scripts/docker/files/leap.list deleted file mode 100644 index 7eb474d8..00000000 --- a/scripts/docker/files/leap.list +++ /dev/null @@ -1,4 +0,0 @@ -# This file is meant to be copied into the `/etc/apt/sources.list.d` directory -# inside a docker image to provide a source for leap-specific packages. - -deb http://deb.leap.se/0.8 jessie main diff --git a/scripts/docker/files/run-perf-test.sh b/scripts/docker/files/run-perf-test.sh deleted file mode 100755 index ebd54d23..00000000 --- a/scripts/docker/files/run-perf-test.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/sh - -# Start a soledad-perf test using a remote server. -# -# The script does the following: -# -# - configure a remote repository for soledad repo if SOLEDAD_REMOTE is set. -# -# - checkout a specific branch if SOLEDAD_BRANCH is set. -# -# - run the soledad-perf local twisted server that runs the client. Note -# that the actual soledad server should be running on another docker -# container. This local server is only used to measure responsiveness of -# soledad client. The script waits for the server to come up before -# continuing, or else times out after TIMEOUT seconds. -# -# - trigger the creation of documents for sync. -# -# - start the measurement of server responsiveness and sync stages. -# -# - stop the test. -# -# This script is meant to be copied to the docker container and run upon -# container start. - -CMD="/usr/local/soledad/test-env.py" -REPO="/var/local/soledad" -TIMEOUT=20 - -#----------------------------------------------------------------------------- -# configure a remote and checkout a branch -#----------------------------------------------------------------------------- - -if [ ! -z "${SOLEDAD_REMOTE}" ]; then - git -C ${REPO} remote set-url origin ${SOLEDAD_REMOTE} - git -C ${REPO} fetch origin -fi - -if [ ! -z "${SOLEDAD_BRANCH}" ]; then - git -C ${REPO} checkout ${SOLEDAD_BRANCH} -fi - -if [ ! -z "${SOLEDAD_PERF_REMOTE}" ]; then - git -C /var/local/soledad-perf remote set-url origin ${SOLEDAD_PERF_REMOTE} - git -C /var/local/soledad-perf fetch origin -fi - -if [ ! -z "${SOLEDAD_PERF_BRANCH}" ]; then - git -C /var/local/soledad-perf checkout ${SOLEDAD_PERF_BRANCH} -fi - -#----------------------------------------------------------------------------- -# write a configuration file for the perf test -#----------------------------------------------------------------------------- - -cd /var/local/soledad-perf - -cat > defaults.conf < /dev/null & -sleep 5 # wait a bit for some data points - -# run a sync and generate a graph -make trigger-sync -make trigger-stop -make graph-image diff --git a/scripts/docker/files/setup-env.sh b/scripts/docker/files/setup-env.sh deleted file mode 100755 index d5aeab7d..00000000 --- a/scripts/docker/files/setup-env.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash - -# Clone soledad repository and install soledad dependencies needed to run -# client and server in a test environment. -# -# In details, this script does the following: -# -# - clone a series of python package repositories into /var/local/soledad. -# - install dependencies for those packages from the requirements files in -# each of the repositories, using python wheels when possible. -# - install the python packages in development mode -# -# The cloned git repositories might have a remote configured and a branch -# checked out on runtime, before a server, client or test instance is actually -# run. Check the other scripts in this directory. -# -# This script is meant to be copied to the docker container and run after -# system dependencies have been installed. - -BASEDIR="/var/local" -BASEURL="https://github.com/leapcode" - -mkdir -p ${BASEDIR} - -# clone repositories -repos="soledad leap_pycommon soledad-perf" - -for repo in ${repos}; do - repodir=${BASEDIR}/${repo} - if [ ! -d ${repodir} ]; then - git clone ${BASEURL}/${repo} ${repodir} - git -C ${repodir} fetch origin - fi -done - -# use latest pip because the version available in debian jessie doesn't -# support wheels -pip install -U pip - -pip install psutil - -# install dependencies and packages -install_script="pkg/pip_install_requirements.sh" -opts="--use-leap-wheels" -pkgs="leap_pycommon soledad/common soledad/client soledad/server" - -for pkg in ${pkgs}; do - pkgdir=${BASEDIR}/${pkg} - testing="" - if [ -f ${pkgdir}/pkg/requirements-testing.pip ]; then - testing="--testing" - fi - (cd ${pkgdir} && ${install_script} ${testing} ${opts}) - (cd ${pkgdir} && python setup.py develop) -done diff --git a/scripts/docker/files/start-client-test.sh b/scripts/docker/files/start-client-test.sh deleted file mode 100755 index 9dec3371..00000000 --- a/scripts/docker/files/start-client-test.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# Run a Soledad client connection test. -# -# This script is meant to be copied to the docker container and run upon -# container start. - -CMD="/usr/local/soledad/test-env.py" -REPO="/var/local/soledad" - -if [ ! -z "${SOLEDAD_REMOTE}" ]; then - git -C ${REPO} remote set-url origin ${SOLEDAD_REMOTE} - git -C ${REPO} fetch origin -fi - -if [ ! -z "${SOLEDAD_BRANCH}" ]; then - git -C ${REPO} checkout ${SOLEDAD_BRANCH} -fi - -${CMD} soledad-client test --server-url ${SOLEDAD_SERVER_URL} diff --git a/scripts/docker/files/start-server.sh b/scripts/docker/files/start-server.sh deleted file mode 100755 index 0980d352..00000000 --- a/scripts/docker/files/start-server.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/sh - -# Start a soledad server inside a docker container. -# -# This script will: -# -# - eventually checkout a specific branch from a specific soledad remote. -# -# - create everything a soledad server needs to run (certificate, backend -# server database, tables, etc. -# -# - eventually preload the server database with a number of documents equal -# to SOLEDAD_PRELOAD_NUM, and with payload size equal to -# SOLEDAD_PRELOAD_SIZE. -# -# - run the soledad server. -# -# This script is meant to be copied to the docker container and run upon -# container start. - -CMD="/usr/local/soledad/test-env.py" - -#--------------------------------------------------------------------------- -# eventually checkout a specific branch from a specific remote -#--------------------------------------------------------------------------- - -REPO="/var/local/soledad" - -if [ ! -z "${SOLEDAD_REMOTE}" ]; then - git -C ${REPO} remote set-url origin ${SOLEDAD_REMOTE} - git -C ${REPO} fetch origin -fi - -if [ ! -z "${SOLEDAD_BRANCH}" ]; then - git -C ${REPO} checkout ${SOLEDAD_BRANCH} -fi - -#--------------------------------------------------------------------------- -# setup environment for running soledad server -#--------------------------------------------------------------------------- - -${CMD} couch start -${CMD} user-db create -${CMD} token-db create -${CMD} token-db insert-token -${CMD} shared-db create -${CMD} cert create - -#--------------------------------------------------------------------------- -# write a configuration file for the perf test -#--------------------------------------------------------------------------- - -if [ "${SOLEDAD_PRELOAD_NUM}" -gt 0 ]; then - cd /var/local/soledad-perf - - cat > defaults.conf < Date: Sun, 12 Jun 2016 13:23:56 -0300 Subject: [test] improve docker scripts README file --- scripts/docker/README.md | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/scripts/docker/README.md b/scripts/docker/README.md index 101db837..c4d7ac94 100644 --- a/scripts/docker/README.md +++ b/scripts/docker/README.md @@ -7,17 +7,18 @@ especially useful for testing purposes as you can limit/reserve a certain amount of resources for the soledad process, and thus provide a baseline for comparison of time and resource consumption between distinct runs. -Check the `Dockerfile` for the rules for building the docker image. +Check the `Dockerfile` for the steps for creating the docker image. -Check the `Makefile` for example usage of the files in this directory. +Check the `Makefile` for the rules for running containers. +Check the `helper/` directory for scripts that help running tests. -Environment variables for server script ---------------------------------------- -If you want to run the image for testing you may pass the following -environment variables for the `files/start-server.sh` script for checking out -a specific branch on the soledad repository: +Environment variables for docker containers +------------------------------------------- + +Different environment variables can be set for docker containers and will +cause the scripts to behave differently: SOLEDAD_REMOTE - a git url for a remote repository that is added at run time to the local soledad git repository. @@ -25,9 +26,24 @@ a specific branch on the soledad repository: SOLEDAD_BRANCH - the name of a branch to be checked out from the configured remote repository. -Example: + SOLEDAD_PRELOAD_NUM - The number of documents to be preloaded in the + container database (either client or server). + + SOLEDAD_PRELOAD_SIZE - The size of the payload of the documents to be + prelaoded in the container database (either client or + server). + + SOLEDAD_SERVER_URL - The URL of the soledad server to be used during the + test. + +Check the Makefile for examples on how to use these and maybe even other +variables not documented here. + + +Communication between client and server containers +-------------------------------------------------- - docker run \ - --env="SOLEDAD_REMOTE=https://0xacab.org/leap/soledad.git" \ - --env="SOLEDAD_BRANCH=develop" \ - leap/soledad:1.0 /usr/local/soledad/start-server.sh +A CONTAINER_ID_FILE variable can be passed to the Makefile target so that the +container id is recorded in a file for further use. This makes it possible to +extract a container's IP and pass it to another container so they can +communicate. -- cgit v1.2.3 From 2c5f5e886a2be8100ef27816adf81ef6cef1545a Mon Sep 17 00:00:00 2001 From: drebs Date: Sun, 12 Jun 2016 13:25:54 -0300 Subject: [test] add logging to client running on docker container --- scripts/docker/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/docker/Makefile b/scripts/docker/Makefile index 9dbe9062..9d669097 100644 --- a/scripts/docker/Makefile +++ b/scripts/docker/Makefile @@ -104,6 +104,7 @@ run-client-perf: --env="SOLEDAD_PRELOAD_SIZE=$(SOLEDAD_PRELOAD_SIZE)" \ --env="SOLEDAD_STATS=1" \ --env="SOLEDAD_SERVER_URL=http://$${server_ip}:2424" \ + --env="SOLEDAD_LOG=1" \ $(IMAGE_NAME) \ /usr/local/soledad/run-client-perf.sh # --drop-to-shell -- cgit v1.2.3 From e433353b877f9cfc735b4368d92b80d091264c65 Mon Sep 17 00:00:00 2001 From: drebs Date: Sun, 12 Jun 2016 13:26:34 -0300 Subject: [test] add commit/branch checkout to docker test running script --- scripts/docker/helper/run-test.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/docker/helper/run-test.sh b/scripts/docker/helper/run-test.sh index c3bdd00c..9b3ec0c9 100755 --- a/scripts/docker/helper/run-test.sh +++ b/scripts/docker/helper/run-test.sh @@ -15,18 +15,22 @@ TIMEOUT=20 # parse command -if [ ${#} -ne 1 ]; then +if [ ${#} -lt 1 -o ${#} -gt 2 ]; then echo "Usage: ${0} perf|bootstrap" exit 1 fi test=${1} - if [ "${test}" != "perf" -a "${test}" != "bootstrap" ]; then echo "Usage: ${0} perf|bootstrap" exit 1 fi +branch="" +if [ ${#} -eq 2 ]; then + branch="SOLEDAD_BRANCH=${2}" +fi + # make sure the image is up to date make image @@ -36,7 +40,7 @@ scriptpath=$(dirname "${script}") # run the server tempfile=`mktemp -u` -make run-server CONTAINER_ID_FILE=${tempfile} +make run-server CONTAINER_ID_FILE=${tempfile} ${branch} # wait for server until timeout container_id=`cat ${tempfile}` @@ -67,5 +71,5 @@ fi set -e # run the test -make run-client-${test} CONTAINER_ID_FILE=${tempfile} +make run-client-${test} CONTAINER_ID_FILE=${tempfile} ${branch} rm -r ${tempfile} -- cgit v1.2.3 From e8d3f659fbd15f7024eba0a9316634fde31b6bdc Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 13 Jun 2016 09:27:23 -0300 Subject: [test] limit cpu sets for docker perf test containers --- scripts/docker/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/docker/Makefile b/scripts/docker/Makefile index 9d669097..4fa2e264 100644 --- a/scripts/docker/Makefile +++ b/scripts/docker/Makefile @@ -43,6 +43,7 @@ run-server: fi docker run \ --memory="$(MEMORY)" \ + --cpuset-cpus=0 \ --env="SOLEDAD_REMOTE=$(SOLEDAD_REMOTE)" \ --env="SOLEDAD_BRANCH=$(SOLEDAD_BRANCH)" \ --env="SOLEDAD_PRELOAD_NUM=$(SOLEDAD_PRELOAD_NUM)" \ @@ -95,6 +96,7 @@ run-client-perf: server_ip=`./helper/get-container-ip.sh $${container_id}`; \ docker run -t -i \ --memory="$(MEMORY)" \ + --cpuset-cpus=1 \ --cidfile=$(CONTAINER_ID_FILE)-perf \ --env="SOLEDAD_REMOTE=$(SOLEDAD_REMOTE)" \ --env="SOLEDAD_BRANCH=$(SOLEDAD_BRANCH)" \ -- cgit v1.2.3 From a973e1ffcb4bce613693a0a3bdcade615cf0b1cb Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 13 Jun 2016 09:28:28 -0300 Subject: [test] use docker for gitlab-ci tests --- .gitlab-ci.yml | 6 ++- .../docker/files/bin/run-trial-from-gitlab-ci.sh | 50 ++++++++++++++++++++++ 2 files changed, 54 insertions(+), 2 deletions(-) create mode 100755 scripts/docker/files/bin/run-trial-from-gitlab-ci.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c6d1998c..1aab6d74 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,3 +1,5 @@ -run_tests: +image: leap/soledad:1.0 + +test: script: - - /home/gitlab-runner/soledad/scripts/gitlab/run_tests.sh + - /usr/local/soledad/run-trial-from-gitlab-ci.sh diff --git a/scripts/docker/files/bin/run-trial-from-gitlab-ci.sh b/scripts/docker/files/bin/run-trial-from-gitlab-ci.sh new file mode 100755 index 00000000..96436e26 --- /dev/null +++ b/scripts/docker/files/bin/run-trial-from-gitlab-ci.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# Run Soledad trial tests in a docker container created by gitlab-ci. +# +# Gitlab-ci will copy the current test code into /builds/leap/soledad, so this +# script has to uninstall currently installed soledad packages and re-install +# from that location instead. +# +# This script is meant to be copied to the docker container and run upon +# container start. + +CMD="/usr/local/soledad/setup-test-env.py" +BASEDIR="/builds/leap/soledad" + + +install_deps() { + # ensure all dependencies are installed + for pkg in common client server; do + testing="--testing" + if [ "${pkg}" = "server" ]; then + # soledad server doesn't currently have a requirements-testing.pip file, + # so we don't pass the option when that is the case + testing="" + fi + pip uninstall leap.soledad.${pkg} + (cd ${BASEDIR}/${pkg} \ + && ./pkg/pip_install_requirements.sh ${testing} --use-leap-wheels \ + && python setup.py develop) + done +} + + +start_couch() { + # currently soledad trial tests need a running couch on environment + ${CMD} couch start +} + + +run_tests() { + trial leap.soledad.common +} + + +main() { + install_deps + start_couch + run_tests +} + +main -- cgit v1.2.3 From 9b3737a66b4df0a6e0fd4d91da097f36e94bd8e2 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 22 Jun 2016 23:09:02 +0200 Subject: pep8 --- common/src/leap/soledad/common/couch/__init__.py | 3 ++- common/src/leap/soledad/common/tests/test_encdecpool.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/common/src/leap/soledad/common/couch/__init__.py b/common/src/leap/soledad/common/couch/__init__.py index 5bda8071..8c60b6a4 100644 --- a/common/src/leap/soledad/common/couch/__init__.py +++ b/common/src/leap/soledad/common/couch/__init__.py @@ -341,7 +341,8 @@ class CouchDatabase(object): # This will not be needed when/if we switch from python-couchdb to # paisley. time.strptime('Mar 8 1917', '%b %d %Y') - get_one = functools.partial(self.get_doc, check_for_conflicts=check_for_conflicts) + get_one = functools.partial( + self.get_doc, check_for_conflicts=check_for_conflicts) docs = [THREAD_POOL.apply_async(get_one, [doc_id]) for doc_id in doc_ids] for doc in docs: diff --git a/common/src/leap/soledad/common/tests/test_encdecpool.py b/common/src/leap/soledad/common/tests/test_encdecpool.py index e6ad66ca..c626561d 100644 --- a/common/src/leap/soledad/common/tests/test_encdecpool.py +++ b/common/src/leap/soledad/common/tests/test_encdecpool.py @@ -133,7 +133,7 @@ class TestSyncDecrypterPool(BaseSoledadTest): self.assertFalse(self._pool.running) self.assertTrue(self._pool.deferred.called) - def test_sync_id_column_is_created_if_non_existing_in_docs_received_table(self): + def test_sync_id_col_is_created_if_non_existing_in_docs_recvd_table(self): """ Test that docs_received table is migrated, and has the sync_id column """ @@ -145,7 +145,8 @@ class TestSyncDecrypterPool(BaseSoledadTest): pool.stop() def assert_trial_to_create_sync_id_column(_): - mock_run_query.assert_called_once_with("ALTER TABLE docs_received ADD COLUMN sync_id") + mock_run_query.assert_called_once_with( + "ALTER TABLE docs_received ADD COLUMN sync_id") d.addCallback(assert_trial_to_create_sync_id_column) return d -- cgit v1.2.3 From 7abf86737562b5c79e902921df722f01e71178e6 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 28 Apr 2016 15:52:30 -0400 Subject: [refactor] fork u1db --- common/src/leap/soledad/common/l2db/__init__.py | 697 ++++++++++++++++ .../leap/soledad/common/l2db/backends/__init__.py | 209 +++++ .../leap/soledad/common/l2db/backends/dbschema.sql | 42 + .../leap/soledad/common/l2db/backends/inmemory.py | 472 +++++++++++ .../soledad/common/l2db/backends/sqlite_backend.py | 926 +++++++++++++++++++++ .../soledad/common/l2db/commandline/__init__.py | 15 + .../leap/soledad/common/l2db/commandline/client.py | 497 +++++++++++ .../soledad/common/l2db/commandline/command.py | 80 ++ .../leap/soledad/common/l2db/commandline/serve.py | 58 ++ common/src/leap/soledad/common/l2db/errors.py | 195 +++++ .../src/leap/soledad/common/l2db/query_parser.py | 370 ++++++++ .../leap/soledad/common/l2db/remote/__init__.py | 15 + .../common/l2db/remote/basic_auth_middleware.py | 68 ++ .../soledad/common/l2db/remote/cors_middleware.py | 42 + .../leap/soledad/common/l2db/remote/http_app.py | 661 +++++++++++++++ .../leap/soledad/common/l2db/remote/http_client.py | 219 +++++ .../soledad/common/l2db/remote/http_database.py | 163 ++++ .../leap/soledad/common/l2db/remote/http_errors.py | 47 ++ .../leap/soledad/common/l2db/remote/http_target.py | 135 +++ .../soledad/common/l2db/remote/oauth_middleware.py | 89 ++ .../soledad/common/l2db/remote/server_state.py | 71 ++ .../common/l2db/remote/ssl_match_hostname.py | 64 ++ .../src/leap/soledad/common/l2db/remote/utils.py | 23 + common/src/leap/soledad/common/l2db/sync.py | 308 +++++++ common/src/leap/soledad/common/l2db/vectorclock.py | 89 ++ 25 files changed, 5555 insertions(+) create mode 100644 common/src/leap/soledad/common/l2db/__init__.py create mode 100644 common/src/leap/soledad/common/l2db/backends/__init__.py create mode 100644 common/src/leap/soledad/common/l2db/backends/dbschema.sql create mode 100644 common/src/leap/soledad/common/l2db/backends/inmemory.py create mode 100644 common/src/leap/soledad/common/l2db/backends/sqlite_backend.py create mode 100644 common/src/leap/soledad/common/l2db/commandline/__init__.py create mode 100644 common/src/leap/soledad/common/l2db/commandline/client.py create mode 100644 common/src/leap/soledad/common/l2db/commandline/command.py create mode 100644 common/src/leap/soledad/common/l2db/commandline/serve.py create mode 100644 common/src/leap/soledad/common/l2db/errors.py create mode 100644 common/src/leap/soledad/common/l2db/query_parser.py create mode 100644 common/src/leap/soledad/common/l2db/remote/__init__.py create mode 100644 common/src/leap/soledad/common/l2db/remote/basic_auth_middleware.py create mode 100644 common/src/leap/soledad/common/l2db/remote/cors_middleware.py create mode 100644 common/src/leap/soledad/common/l2db/remote/http_app.py create mode 100644 common/src/leap/soledad/common/l2db/remote/http_client.py create mode 100644 common/src/leap/soledad/common/l2db/remote/http_database.py create mode 100644 common/src/leap/soledad/common/l2db/remote/http_errors.py create mode 100644 common/src/leap/soledad/common/l2db/remote/http_target.py create mode 100644 common/src/leap/soledad/common/l2db/remote/oauth_middleware.py create mode 100644 common/src/leap/soledad/common/l2db/remote/server_state.py create mode 100644 common/src/leap/soledad/common/l2db/remote/ssl_match_hostname.py create mode 100644 common/src/leap/soledad/common/l2db/remote/utils.py create mode 100644 common/src/leap/soledad/common/l2db/sync.py create mode 100644 common/src/leap/soledad/common/l2db/vectorclock.py diff --git a/common/src/leap/soledad/common/l2db/__init__.py b/common/src/leap/soledad/common/l2db/__init__.py new file mode 100644 index 00000000..e33309a4 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/__init__.py @@ -0,0 +1,697 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +"""U1DB""" + +try: + import simplejson as json +except ImportError: + import json # noqa + +from u1db.errors import InvalidJSON, InvalidContent + +__version_info__ = (13, 9) +__version__ = '.'.join(map(lambda x: '%02d' % x, __version_info__)) + + +def open(path, create, document_factory=None): + """Open a database at the given location. + + Will raise u1db.errors.DatabaseDoesNotExist if create=False and the + database does not already exist. + + :param path: The filesystem path for the database to open. + :param create: True/False, should the database be created if it doesn't + already exist? + :param document_factory: A function that will be called with the same + parameters as Document.__init__. + :return: An instance of Database. + """ + from u1db.backends import sqlite_backend + return sqlite_backend.SQLiteDatabase.open_database( + path, create=create, document_factory=document_factory) + + +# constraints on database names (relevant for remote access, as regex) +DBNAME_CONSTRAINTS = r"[a-zA-Z0-9][a-zA-Z0-9.-]*" + +# constraints on doc ids (as regex) +# (no slashes, and no characters outside the ascii range) +DOC_ID_CONSTRAINTS = r"[a-zA-Z0-9.%_-]+" + + +class Database(object): + """A JSON Document data store. + + This data store can be synchronized with other u1db.Database instances. + """ + + def set_document_factory(self, factory): + """Set the document factory that will be used to create objects to be + returned as documents by the database. + + :param factory: A function that returns an object which at minimum must + satisfy the same interface as does the class DocumentBase. + Subclassing that class is the easiest way to create such + a function. + """ + raise NotImplementedError(self.set_document_factory) + + def set_document_size_limit(self, limit): + """Set the maximum allowed document size for this database. + + :param limit: Maximum allowed document size in bytes. + """ + raise NotImplementedError(self.set_document_size_limit) + + def whats_changed(self, old_generation=0): + """Return a list of documents that have changed since old_generation. + This allows APPS to only store a db generation before going + 'offline', and then when coming back online they can use this + data to update whatever extra data they are storing. + + :param old_generation: The generation of the database in the old + state. + :return: (generation, trans_id, [(doc_id, generation, trans_id),...]) + The current generation of the database, its associated transaction + id, and a list of of changed documents since old_generation, + represented by tuples with for each document its doc_id and the + generation and transaction id corresponding to the last intervening + change and sorted by generation (old changes first) + """ + raise NotImplementedError(self.whats_changed) + + def get_doc(self, doc_id, include_deleted=False): + """Get the JSON string for the given document. + + :param doc_id: The unique document identifier + :param include_deleted: If set to True, deleted documents will be + returned with empty content. Otherwise asking for a deleted + document will return None. + :return: a Document object. + """ + raise NotImplementedError(self.get_doc) + + def get_docs(self, doc_ids, check_for_conflicts=True, + include_deleted=False): + """Get the JSON content for many documents. + + :param doc_ids: A list of document identifiers. + :param check_for_conflicts: If set to False, then the conflict check + will be skipped, and 'None' will be returned instead of True/False. + :param include_deleted: If set to True, deleted documents will be + returned with empty content. Otherwise deleted documents will not + be included in the results. + :return: iterable giving the Document object for each document id + in matching doc_ids order. + """ + raise NotImplementedError(self.get_docs) + + def get_all_docs(self, include_deleted=False): + """Get the JSON content for all documents in the database. + + :param include_deleted: If set to True, deleted documents will be + returned with empty content. Otherwise deleted documents will not + be included in the results. + :return: (generation, [Document]) + The current generation of the database, followed by a list of all + the documents in the database. + """ + raise NotImplementedError(self.get_all_docs) + + def create_doc(self, content, doc_id=None): + """Create a new document. + + You can optionally specify the document identifier, but the document + must not already exist. See 'put_doc' if you want to override an + existing document. + If the database specifies a maximum document size and the document + exceeds it, create will fail and raise a DocumentTooBig exception. + + :param content: A Python dictionary. + :param doc_id: An optional identifier specifying the document id. + :return: Document + """ + raise NotImplementedError(self.create_doc) + + def create_doc_from_json(self, json, doc_id=None): + """Create a new document. + + You can optionally specify the document identifier, but the document + must not already exist. See 'put_doc' if you want to override an + existing document. + If the database specifies a maximum document size and the document + exceeds it, create will fail and raise a DocumentTooBig exception. + + :param json: The JSON document string + :param doc_id: An optional identifier specifying the document id. + :return: Document + """ + raise NotImplementedError(self.create_doc_from_json) + + def put_doc(self, doc): + """Update a document. + If the document currently has conflicts, put will fail. + If the database specifies a maximum document size and the document + exceeds it, put will fail and raise a DocumentTooBig exception. + + :param doc: A Document with new content. + :return: new_doc_rev - The new revision identifier for the document. + The Document object will also be updated. + """ + raise NotImplementedError(self.put_doc) + + def delete_doc(self, doc): + """Mark a document as deleted. + Will abort if the current revision doesn't match doc.rev. + This will also set doc.content to None. + """ + raise NotImplementedError(self.delete_doc) + + def create_index(self, index_name, *index_expressions): + """Create an named index, which can then be queried for future lookups. + Creating an index which already exists is not an error, and is cheap. + Creating an index which does not match the index_expressions of the + existing index is an error. + Creating an index will block until the expressions have been evaluated + and the index generated. + + :param index_name: A unique name which can be used as a key prefix + :param index_expressions: index expressions defining the index + information. + + Examples: + + "fieldname", or "fieldname.subfieldname" to index alphabetically + sorted on the contents of a field. + + "number(fieldname, width)", "lower(fieldname)" + """ + raise NotImplementedError(self.create_index) + + def delete_index(self, index_name): + """Remove a named index. + + :param index_name: The name of the index we are removing + """ + raise NotImplementedError(self.delete_index) + + def list_indexes(self): + """List the definitions of all known indexes. + + :return: A list of [('index-name', ['field', 'field2'])] definitions. + """ + raise NotImplementedError(self.list_indexes) + + def get_from_index(self, index_name, *key_values): + """Return documents that match the keys supplied. + + You must supply exactly the same number of values as have been defined + in the index. It is possible to do a prefix match by using '*' to + indicate a wildcard match. You can only supply '*' to trailing entries, + (eg 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) + It is also possible to append a '*' to the last supplied value (eg + 'val*', '*', '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') + + :param index_name: The index to query + :param key_values: values to match. eg, if you have + an index with 3 fields then you would have: + get_from_index(index_name, val1, val2, val3) + :return: List of [Document] + """ + raise NotImplementedError(self.get_from_index) + + def get_range_from_index(self, index_name, start_value, end_value): + """Return documents that fall within the specified range. + + Both ends of the range are inclusive. For both start_value and + end_value, one must supply exactly the same number of values as have + been defined in the index, or pass None. In case of a single column + index, a string is accepted as an alternative for a tuple with a single + value. It is possible to do a prefix match by using '*' to indicate + a wildcard match. You can only supply '*' to trailing entries, (eg + 'val', '*', '*' is allowed, but '*', 'val', 'val' is not.) It is also + possible to append a '*' to the last supplied value (eg 'val*', '*', + '*' or 'val', 'val*', '*', but not 'val*', 'val', '*') + + :param index_name: The index to query + :param start_values: tuples of values that define the lower bound of + the range. eg, if you have an index with 3 fields then you would + have: (val1, val2, val3) + :param end_values: tuples of values that define the upper bound of the + range. eg, if you have an index with 3 fields then you would have: + (val1, val2, val3) + :return: List of [Document] + """ + raise NotImplementedError(self.get_range_from_index) + + def get_index_keys(self, index_name): + """Return all keys under which documents are indexed in this index. + + :param index_name: The index to query + :return: [] A list of tuples of indexed keys. + """ + raise NotImplementedError(self.get_index_keys) + + def get_doc_conflicts(self, doc_id): + """Get the list of conflicts for the given document. + + The order of the conflicts is such that the first entry is the value + that would be returned by "get_doc". + + :return: [doc] A list of the Document entries that are conflicted. + """ + raise NotImplementedError(self.get_doc_conflicts) + + def resolve_doc(self, doc, conflicted_doc_revs): + """Mark a document as no longer conflicted. + + We take the list of revisions that the client knows about that it is + superseding. This may be a different list from the actual current + conflicts, in which case only those are removed as conflicted. This + may fail if the conflict list is significantly different from the + supplied information. (sync could have happened in the background from + the time you GET_DOC_CONFLICTS until the point where you RESOLVE) + + :param doc: A Document with the new content to be inserted. + :param conflicted_doc_revs: A list of revisions that the new content + supersedes. + """ + raise NotImplementedError(self.resolve_doc) + + def get_sync_target(self): + """Return a SyncTarget object, for another u1db to synchronize with. + + :return: An instance of SyncTarget. + """ + raise NotImplementedError(self.get_sync_target) + + def close(self): + """Release any resources associated with this database.""" + raise NotImplementedError(self.close) + + def sync(self, url, creds=None, autocreate=True): + """Synchronize documents with remote replica exposed at url. + + :param url: the url of the target replica to sync with. + :param creds: optional dictionary giving credentials + to authorize the operation with the server. For using OAuth + the form of creds is: + {'oauth': { + 'consumer_key': ..., + 'consumer_secret': ..., + 'token_key': ..., + 'token_secret': ... + }} + :param autocreate: ask the target to create the db if non-existent. + :return: local_gen_before_sync The local generation before the + synchronisation was performed. This is useful to pass into + whatschanged, if an application wants to know which documents were + affected by a synchronisation. + """ + from u1db.sync import Synchronizer + from u1db.remote.http_target import HTTPSyncTarget + return Synchronizer(self, HTTPSyncTarget(url, creds=creds)).sync( + autocreate=autocreate) + + def _get_replica_gen_and_trans_id(self, other_replica_uid): + """Return the last known generation and transaction id for the other db + replica. + + When you do a synchronization with another replica, the Database keeps + track of what generation the other database replica was at, and what + the associated transaction id was. This is used to determine what data + needs to be sent, and if two databases are claiming to be the same + replica. + + :param other_replica_uid: The identifier for the other replica. + :return: (gen, trans_id) The generation and transaction id we + encountered during synchronization. If we've never synchronized + with the replica, this is (0, ''). + """ + raise NotImplementedError(self._get_replica_gen_and_trans_id) + + def _set_replica_gen_and_trans_id(self, other_replica_uid, + other_generation, other_transaction_id): + """Set the last-known generation and transaction id for the other + database replica. + + We have just performed some synchronization, and we want to track what + generation the other replica was at. See also + _get_replica_gen_and_trans_id. + :param other_replica_uid: The U1DB identifier for the other replica. + :param other_generation: The generation number for the other replica. + :param other_transaction_id: The transaction id associated with the + generation. + """ + raise NotImplementedError(self._set_replica_gen_and_trans_id) + + def _put_doc_if_newer(self, doc, save_conflict, replica_uid, replica_gen, + replica_trans_id=''): + """Insert/update document into the database with a given revision. + + This api is used during synchronization operations. + + If a document would conflict and save_conflict is set to True, the + content will be selected as the 'current' content for doc.doc_id, + even though doc.rev doesn't supersede the currently stored revision. + The currently stored document will be added to the list of conflict + alternatives for the given doc_id. + + This forces the new content to be 'current' so that we get convergence + after synchronizing, even if people don't resolve conflicts. Users can + then notice that their content is out of date, update it, and + synchronize again. (The alternative is that users could synchronize and + think the data has propagated, but their local copy looks fine, and the + remote copy is never updated again.) + + :param doc: A Document object + :param save_conflict: If this document is a conflict, do you want to + save it as a conflict, or just ignore it. + :param replica_uid: A unique replica identifier. + :param replica_gen: The generation of the replica corresponding to the + this document. The replica arguments are optional, but are used + during synchronization. + :param replica_trans_id: The transaction_id associated with the + generation. + :return: (state, at_gen) - If we don't have doc_id already, + or if doc_rev supersedes the existing document revision, + then the content will be inserted, and state is 'inserted'. + If doc_rev is less than or equal to the existing revision, + then the put is ignored and state is respecitvely 'superseded' + or 'converged'. + If doc_rev is not strictly superseded or supersedes, then + state is 'conflicted'. The document will not be inserted if + save_conflict is False. + For 'inserted' or 'converged', at_gen is the insertion/current + generation. + """ + raise NotImplementedError(self._put_doc_if_newer) + + +class DocumentBase(object): + """Container for handling a single document. + + :ivar doc_id: Unique identifier for this document. + :ivar rev: The revision identifier of the document. + :ivar json_string: The JSON string for this document. + :ivar has_conflicts: Boolean indicating if this document has conflicts + """ + + def __init__(self, doc_id, rev, json_string, has_conflicts=False): + self.doc_id = doc_id + self.rev = rev + if json_string is not None: + try: + value = json.loads(json_string) + except ValueError: + raise InvalidJSON + if not isinstance(value, dict): + raise InvalidJSON + self._json = json_string + self.has_conflicts = has_conflicts + + def same_content_as(self, other): + """Compare the content of two documents.""" + if self._json: + c1 = json.loads(self._json) + else: + c1 = None + if other._json: + c2 = json.loads(other._json) + else: + c2 = None + return c1 == c2 + + def __repr__(self): + if self.has_conflicts: + extra = ', conflicted' + else: + extra = '' + return '%s(%s, %s%s, %r)' % (self.__class__.__name__, self.doc_id, + self.rev, extra, self.get_json()) + + def __hash__(self): + raise NotImplementedError(self.__hash__) + + def __eq__(self, other): + if not isinstance(other, Document): + return NotImplemented + return ( + self.doc_id == other.doc_id and self.rev == other.rev and + self.same_content_as(other) and self.has_conflicts == + other.has_conflicts) + + def __lt__(self, other): + """This is meant for testing, not part of the official api. + + It is implemented so that sorted([Document, Document]) can be used. + It doesn't imply that users would want their documents to be sorted in + this order. + """ + # Since this is just for testing, we don't worry about comparing + # against things that aren't a Document. + return ((self.doc_id, self.rev, self.get_json()) + < (other.doc_id, other.rev, other.get_json())) + + def get_json(self): + """Get the json serialization of this document.""" + if self._json is not None: + return self._json + return None + + def get_size(self): + """Calculate the total size of the document.""" + size = 0 + json = self.get_json() + if json: + size += len(json) + if self.rev: + size += len(self.rev) + if self.doc_id: + size += len(self.doc_id) + return size + + def set_json(self, json_string): + """Set the json serialization of this document.""" + if json_string is not None: + try: + value = json.loads(json_string) + except ValueError: + raise InvalidJSON + if not isinstance(value, dict): + raise InvalidJSON + self._json = json_string + + def make_tombstone(self): + """Make this document into a tombstone.""" + self._json = None + + def is_tombstone(self): + """Return True if the document is a tombstone, False otherwise.""" + if self._json is not None: + return False + return True + + +class Document(DocumentBase): + """Container for handling a single document. + + :ivar doc_id: Unique identifier for this document. + :ivar rev: The revision identifier of the document. + :ivar json: The JSON string for this document. + :ivar has_conflicts: Boolean indicating if this document has conflicts + """ + + # The following part of the API is optional: no implementation is forced to + # have it but if the language supports dictionaries/hashtables, it makes + # Documents a lot more user friendly. + + def __init__(self, doc_id=None, rev=None, json='{}', has_conflicts=False): + # TODO: We convert the json in the superclass to check its validity so + # we might as well set _content here directly since the price is + # already being paid. + super(Document, self).__init__(doc_id, rev, json, has_conflicts) + self._content = None + + def same_content_as(self, other): + """Compare the content of two documents.""" + if self._json: + c1 = json.loads(self._json) + else: + c1 = self._content + if other._json: + c2 = json.loads(other._json) + else: + c2 = other._content + return c1 == c2 + + def get_json(self): + """Get the json serialization of this document.""" + json_string = super(Document, self).get_json() + if json_string is not None: + return json_string + if self._content is not None: + return json.dumps(self._content) + return None + + def set_json(self, json): + """Set the json serialization of this document.""" + self._content = None + super(Document, self).set_json(json) + + def make_tombstone(self): + """Make this document into a tombstone.""" + self._content = None + super(Document, self).make_tombstone() + + def is_tombstone(self): + """Return True if the document is a tombstone, False otherwise.""" + if self._content is not None: + return False + return super(Document, self).is_tombstone() + + def _get_content(self): + """Get the dictionary representing this document.""" + if self._json is not None: + self._content = json.loads(self._json) + self._json = None + if self._content is not None: + return self._content + return None + + def _set_content(self, content): + """Set the dictionary representing this document.""" + try: + tmp = json.dumps(content) + except TypeError: + raise InvalidContent( + "Can not be converted to JSON: %r" % (content,)) + if not tmp.startswith('{'): + raise InvalidContent( + "Can not be converted to a JSON object: %r." % (content,)) + # We might as well store the JSON at this point since we did the work + # of encoding it, and it doesn't lose any information. + self._json = tmp + self._content = None + + content = property( + _get_content, _set_content, doc="Content of the Document.") + + # End of optional part. + + +class SyncTarget(object): + """Functionality for using a Database as a synchronization target.""" + + def get_sync_info(self, source_replica_uid): + """Return information about known state. + + Return the replica_uid and the current database generation of this + database, and the last-seen database generation for source_replica_uid + + :param source_replica_uid: Another replica which we might have + synchronized with in the past. + :return: (target_replica_uid, target_replica_generation, + target_trans_id, source_replica_last_known_generation, + source_replica_last_known_transaction_id) + """ + raise NotImplementedError(self.get_sync_info) + + def record_sync_info(self, source_replica_uid, source_replica_generation, + source_replica_transaction_id): + """Record tip information for another replica. + + After sync_exchange has been processed, the caller will have + received new content from this replica. This call allows the + source replica instigating the sync to inform us what their + generation became after applying the documents we returned. + + This is used to allow future sync operations to not need to repeat data + that we just talked about. It also means that if this is called at the + wrong time, there can be database records that will never be + synchronized. + + :param source_replica_uid: The identifier for the source replica. + :param source_replica_generation: + The database generation for the source replica. + :param source_replica_transaction_id: The transaction id associated + with the source replica generation. + """ + raise NotImplementedError(self.record_sync_info) + + def sync_exchange(self, docs_by_generation, source_replica_uid, + last_known_generation, last_known_trans_id, + return_doc_cb, ensure_callback=None): + """Incorporate the documents sent from the source replica. + + This is not meant to be called by client code directly, but is used as + part of sync(). + + This adds docs to the local store, and determines documents that need + to be returned to the source replica. + + Documents must be supplied in docs_by_generation paired with + the generation of their latest change in order from the oldest + change to the newest, that means from the oldest generation to + the newest. + + Documents are also returned paired with the generation of + their latest change in order from the oldest change to the + newest. + + :param docs_by_generation: A list of [(Document, generation, + transaction_id)] tuples indicating documents which should be + updated on this replica paired with the generation and transaction + id of their latest change. + :param source_replica_uid: The source replica's identifier + :param last_known_generation: The last generation that the source + replica knows about this target replica + :param last_known_trans_id: The last transaction id that the source + replica knows about this target replica + :param: return_doc_cb(doc, gen): is a callback + used to return documents to the source replica, it will + be invoked in turn with Documents that have changed since + last_known_generation together with the generation of + their last change. + :param: ensure_callback(replica_uid): if set the target may create + the target db if not yet existent, the callback can then + be used to inform of the created db replica uid. + :return: new_generation - After applying docs_by_generation, this is + the current generation for this replica + """ + raise NotImplementedError(self.sync_exchange) + + def _set_trace_hook(self, cb): + """Set a callback that will be invoked to trace database actions. + + The callback will be passed a string indicating the current state, and + the sync target object. Implementations do not have to implement this + api, it is used by the test suite. + + :param cb: A callable that takes cb(state) + """ + raise NotImplementedError(self._set_trace_hook) + + def _set_trace_hook_shallow(self, cb): + """Set a callback that will be invoked to trace database actions. + + Similar to _set_trace_hook, for implementations that don't offer + state changes from the inner working of sync_exchange(). + + :param cb: A callable that takes cb(state) + """ + self._set_trace_hook(cb) diff --git a/common/src/leap/soledad/common/l2db/backends/__init__.py b/common/src/leap/soledad/common/l2db/backends/__init__.py new file mode 100644 index 00000000..a647c8aa --- /dev/null +++ b/common/src/leap/soledad/common/l2db/backends/__init__.py @@ -0,0 +1,209 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +"""Abstract classes and common implementations for the backends.""" + +import re +try: + import simplejson as json +except ImportError: + import json # noqa +import uuid + +import u1db +from u1db import ( + errors, +) +import u1db.sync +from u1db.vectorclock import VectorClockRev + + +check_doc_id_re = re.compile("^" + u1db.DOC_ID_CONSTRAINTS + "$", re.UNICODE) + + +class CommonSyncTarget(u1db.sync.LocalSyncTarget): + pass + + +class CommonBackend(u1db.Database): + + document_size_limit = 0 + + def _allocate_doc_id(self): + """Generate a unique identifier for this document.""" + return 'D-' + uuid.uuid4().hex # 'D-' stands for document + + def _allocate_transaction_id(self): + return 'T-' + uuid.uuid4().hex # 'T-' stands for transaction + + def _allocate_doc_rev(self, old_doc_rev): + vcr = VectorClockRev(old_doc_rev) + vcr.increment(self._replica_uid) + return vcr.as_str() + + def _check_doc_id(self, doc_id): + if not check_doc_id_re.match(doc_id): + raise errors.InvalidDocId() + + def _check_doc_size(self, doc): + if not self.document_size_limit: + return + if doc.get_size() > self.document_size_limit: + raise errors.DocumentTooBig + + def _get_generation(self): + """Return the current generation. + + """ + raise NotImplementedError(self._get_generation) + + def _get_generation_info(self): + """Return the current generation and transaction id. + + """ + raise NotImplementedError(self._get_generation_info) + + def _get_doc(self, doc_id, check_for_conflicts=False): + """Extract the document from storage. + + This can return None if the document doesn't exist. + """ + raise NotImplementedError(self._get_doc) + + def _has_conflicts(self, doc_id): + """Return True if the doc has conflicts, False otherwise.""" + raise NotImplementedError(self._has_conflicts) + + def create_doc(self, content, doc_id=None): + if not isinstance(content, dict): + raise errors.InvalidContent + json_string = json.dumps(content) + return self.create_doc_from_json(json_string, doc_id) + + def create_doc_from_json(self, json, doc_id=None): + if doc_id is None: + doc_id = self._allocate_doc_id() + doc = self._factory(doc_id, None, json) + self.put_doc(doc) + return doc + + def _get_transaction_log(self): + """This is only for the test suite, it is not part of the api.""" + raise NotImplementedError(self._get_transaction_log) + + def _put_and_update_indexes(self, doc_id, old_doc, new_rev, content): + raise NotImplementedError(self._put_and_update_indexes) + + def get_docs(self, doc_ids, check_for_conflicts=True, + include_deleted=False): + for doc_id in doc_ids: + doc = self._get_doc( + doc_id, check_for_conflicts=check_for_conflicts) + if doc.is_tombstone() and not include_deleted: + continue + yield doc + + def _get_trans_id_for_gen(self, generation): + """Get the transaction id corresponding to a particular generation. + + Raises an InvalidGeneration when the generation does not exist. + + """ + raise NotImplementedError(self._get_trans_id_for_gen) + + def validate_gen_and_trans_id(self, generation, trans_id): + """Validate the generation and transaction id. + + Raises an InvalidGeneration when the generation does not exist, and an + InvalidTransactionId when it does but with a different transaction id. + + """ + if generation == 0: + return + known_trans_id = self._get_trans_id_for_gen(generation) + if known_trans_id != trans_id: + raise errors.InvalidTransactionId + + def _validate_source(self, other_replica_uid, other_generation, + other_transaction_id): + """Validate the new generation and transaction id. + + other_generation must be greater than what we have stored for this + replica, *or* it must be the same and the transaction_id must be the + same as well. + """ + (old_generation, + old_transaction_id) = self._get_replica_gen_and_trans_id( + other_replica_uid) + if other_generation < old_generation: + raise errors.InvalidGeneration + if other_generation > old_generation: + return + if other_transaction_id == old_transaction_id: + return + raise errors.InvalidTransactionId + + def _put_doc_if_newer(self, doc, save_conflict, replica_uid, replica_gen, + replica_trans_id=''): + cur_doc = self._get_doc(doc.doc_id) + doc_vcr = VectorClockRev(doc.rev) + if cur_doc is None: + cur_vcr = VectorClockRev(None) + else: + cur_vcr = VectorClockRev(cur_doc.rev) + self._validate_source(replica_uid, replica_gen, replica_trans_id) + if doc_vcr.is_newer(cur_vcr): + rev = doc.rev + self._prune_conflicts(doc, doc_vcr) + if doc.rev != rev: + # conflicts have been autoresolved + state = 'superseded' + else: + state = 'inserted' + self._put_and_update_indexes(cur_doc, doc) + elif doc.rev == cur_doc.rev: + # magical convergence + state = 'converged' + elif cur_vcr.is_newer(doc_vcr): + # Don't add this to seen_ids, because we have something newer, + # so we should send it back, and we should not generate a + # conflict + state = 'superseded' + elif cur_doc.same_content_as(doc): + # the documents have been edited to the same thing at both ends + doc_vcr.maximize(cur_vcr) + doc_vcr.increment(self._replica_uid) + doc.rev = doc_vcr.as_str() + self._put_and_update_indexes(cur_doc, doc) + state = 'superseded' + else: + state = 'conflicted' + if save_conflict: + self._force_doc_sync_conflict(doc) + if replica_uid is not None and replica_gen is not None: + self._do_set_replica_gen_and_trans_id( + replica_uid, replica_gen, replica_trans_id) + return state, self._get_generation() + + def _ensure_maximal_rev(self, cur_rev, extra_revs): + vcr = VectorClockRev(cur_rev) + for rev in extra_revs: + vcr.maximize(VectorClockRev(rev)) + vcr.increment(self._replica_uid) + return vcr.as_str() + + def set_document_size_limit(self, limit): + self.document_size_limit = limit diff --git a/common/src/leap/soledad/common/l2db/backends/dbschema.sql b/common/src/leap/soledad/common/l2db/backends/dbschema.sql new file mode 100644 index 00000000..ae027fc5 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/backends/dbschema.sql @@ -0,0 +1,42 @@ +-- Database schema +CREATE TABLE transaction_log ( + generation INTEGER PRIMARY KEY AUTOINCREMENT, + doc_id TEXT NOT NULL, + transaction_id TEXT NOT NULL +); +CREATE TABLE document ( + doc_id TEXT PRIMARY KEY, + doc_rev TEXT NOT NULL, + content TEXT +); +CREATE TABLE document_fields ( + doc_id TEXT NOT NULL, + field_name TEXT NOT NULL, + value TEXT +); +CREATE INDEX document_fields_field_value_doc_idx + ON document_fields(field_name, value, doc_id); + +CREATE TABLE sync_log ( + replica_uid TEXT PRIMARY KEY, + known_generation INTEGER, + known_transaction_id TEXT +); +CREATE TABLE conflicts ( + doc_id TEXT, + doc_rev TEXT, + content TEXT, + CONSTRAINT conflicts_pkey PRIMARY KEY (doc_id, doc_rev) +); +CREATE TABLE index_definitions ( + name TEXT, + offset INT, + field TEXT, + CONSTRAINT index_definitions_pkey PRIMARY KEY (name, offset) +); +create index index_definitions_field on index_definitions(field); +CREATE TABLE u1db_config ( + name TEXT PRIMARY KEY, + value TEXT +); +INSERT INTO u1db_config VALUES ('sql_schema', '0'); diff --git a/common/src/leap/soledad/common/l2db/backends/inmemory.py b/common/src/leap/soledad/common/l2db/backends/inmemory.py new file mode 100644 index 00000000..1feb1604 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/backends/inmemory.py @@ -0,0 +1,472 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +"""The in-memory Database class for U1DB.""" + +try: + import simplejson as json +except ImportError: + import json # noqa + +from u1db import ( + Document, + errors, + query_parser, + vectorclock, + ) +from u1db.backends import CommonBackend, CommonSyncTarget + + +def get_prefix(value): + key_prefix = '\x01'.join(value) + return key_prefix.rstrip('*') + + +class InMemoryDatabase(CommonBackend): + """A database that only stores the data internally.""" + + def __init__(self, replica_uid, document_factory=None): + self._transaction_log = [] + self._docs = {} + # Map from doc_id => [(doc_rev, doc)] conflicts beyond 'winner' + self._conflicts = {} + self._other_generations = {} + self._indexes = {} + self._replica_uid = replica_uid + self._factory = document_factory or Document + + def _set_replica_uid(self, replica_uid): + """Force the replica_uid to be set.""" + self._replica_uid = replica_uid + + def set_document_factory(self, factory): + self._factory = factory + + def close(self): + # This is a no-op, We don't want to free the data because one client + # may be closing it, while another wants to inspect the results. + pass + + def _get_replica_gen_and_trans_id(self, other_replica_uid): + return self._other_generations.get(other_replica_uid, (0, '')) + + def _set_replica_gen_and_trans_id(self, other_replica_uid, + other_generation, other_transaction_id): + self._do_set_replica_gen_and_trans_id( + other_replica_uid, other_generation, other_transaction_id) + + def _do_set_replica_gen_and_trans_id(self, other_replica_uid, + other_generation, + other_transaction_id): + # TODO: to handle race conditions, we may want to check if the current + # value is greater than this new value. + self._other_generations[other_replica_uid] = (other_generation, + other_transaction_id) + + def get_sync_target(self): + return InMemorySyncTarget(self) + + def _get_transaction_log(self): + # snapshot! + return self._transaction_log[:] + + def _get_generation(self): + return len(self._transaction_log) + + def _get_generation_info(self): + if not self._transaction_log: + return 0, '' + return len(self._transaction_log), self._transaction_log[-1][1] + + def _get_trans_id_for_gen(self, generation): + if generation == 0: + return '' + if generation > len(self._transaction_log): + raise errors.InvalidGeneration + return self._transaction_log[generation - 1][1] + + def put_doc(self, doc): + if doc.doc_id is None: + raise errors.InvalidDocId() + self._check_doc_id(doc.doc_id) + self._check_doc_size(doc) + old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True) + if old_doc and old_doc.has_conflicts: + raise errors.ConflictedDoc() + if old_doc and doc.rev is None and old_doc.is_tombstone(): + new_rev = self._allocate_doc_rev(old_doc.rev) + else: + if old_doc is not None: + if old_doc.rev != doc.rev: + raise errors.RevisionConflict() + else: + if doc.rev is not None: + raise errors.RevisionConflict() + new_rev = self._allocate_doc_rev(doc.rev) + doc.rev = new_rev + self._put_and_update_indexes(old_doc, doc) + return new_rev + + def _put_and_update_indexes(self, old_doc, doc): + for index in self._indexes.itervalues(): + if old_doc is not None and not old_doc.is_tombstone(): + index.remove_json(old_doc.doc_id, old_doc.get_json()) + if not doc.is_tombstone(): + index.add_json(doc.doc_id, doc.get_json()) + trans_id = self._allocate_transaction_id() + self._docs[doc.doc_id] = (doc.rev, doc.get_json()) + self._transaction_log.append((doc.doc_id, trans_id)) + + def _get_doc(self, doc_id, check_for_conflicts=False): + try: + doc_rev, content = self._docs[doc_id] + except KeyError: + return None + doc = self._factory(doc_id, doc_rev, content) + if check_for_conflicts: + doc.has_conflicts = (doc.doc_id in self._conflicts) + return doc + + def _has_conflicts(self, doc_id): + return doc_id in self._conflicts + + def get_doc(self, doc_id, include_deleted=False): + doc = self._get_doc(doc_id, check_for_conflicts=True) + if doc is None: + return None + if doc.is_tombstone() and not include_deleted: + return None + return doc + + def get_all_docs(self, include_deleted=False): + """Return all documents in the database.""" + generation = self._get_generation() + results = [] + for doc_id, (doc_rev, content) in self._docs.items(): + if content is None and not include_deleted: + continue + doc = self._factory(doc_id, doc_rev, content) + doc.has_conflicts = self._has_conflicts(doc_id) + results.append(doc) + return (generation, results) + + def get_doc_conflicts(self, doc_id): + if doc_id not in self._conflicts: + return [] + result = [self._get_doc(doc_id)] + result[0].has_conflicts = True + result.extend([self._factory(doc_id, rev, content) + for rev, content in self._conflicts[doc_id]]) + return result + + def _replace_conflicts(self, doc, conflicts): + if not conflicts: + del self._conflicts[doc.doc_id] + else: + self._conflicts[doc.doc_id] = conflicts + doc.has_conflicts = bool(conflicts) + + def _prune_conflicts(self, doc, doc_vcr): + if self._has_conflicts(doc.doc_id): + autoresolved = False + remaining_conflicts = [] + cur_conflicts = self._conflicts[doc.doc_id] + for c_rev, c_doc in cur_conflicts: + c_vcr = vectorclock.VectorClockRev(c_rev) + if doc_vcr.is_newer(c_vcr): + continue + if doc.same_content_as(Document(doc.doc_id, c_rev, c_doc)): + doc_vcr.maximize(c_vcr) + autoresolved = True + continue + remaining_conflicts.append((c_rev, c_doc)) + if autoresolved: + doc_vcr.increment(self._replica_uid) + doc.rev = doc_vcr.as_str() + self._replace_conflicts(doc, remaining_conflicts) + + def resolve_doc(self, doc, conflicted_doc_revs): + cur_doc = self._get_doc(doc.doc_id) + if cur_doc is None: + cur_rev = None + else: + cur_rev = cur_doc.rev + new_rev = self._ensure_maximal_rev(cur_rev, conflicted_doc_revs) + superseded_revs = set(conflicted_doc_revs) + remaining_conflicts = [] + cur_conflicts = self._conflicts[doc.doc_id] + for c_rev, c_doc in cur_conflicts: + if c_rev in superseded_revs: + continue + remaining_conflicts.append((c_rev, c_doc)) + doc.rev = new_rev + if cur_rev in superseded_revs: + self._put_and_update_indexes(cur_doc, doc) + else: + remaining_conflicts.append((new_rev, doc.get_json())) + self._replace_conflicts(doc, remaining_conflicts) + + def delete_doc(self, doc): + if doc.doc_id not in self._docs: + raise errors.DocumentDoesNotExist + if self._docs[doc.doc_id][1] in ('null', None): + raise errors.DocumentAlreadyDeleted + doc.make_tombstone() + self.put_doc(doc) + + def create_index(self, index_name, *index_expressions): + if index_name in self._indexes: + if self._indexes[index_name]._definition == list( + index_expressions): + return + raise errors.IndexNameTakenError + index = InMemoryIndex(index_name, list(index_expressions)) + for doc_id, (doc_rev, doc) in self._docs.iteritems(): + if doc is not None: + index.add_json(doc_id, doc) + self._indexes[index_name] = index + + def delete_index(self, index_name): + try: + del self._indexes[index_name] + except KeyError: + pass + + def list_indexes(self): + definitions = [] + for idx in self._indexes.itervalues(): + definitions.append((idx._name, idx._definition)) + return definitions + + def get_from_index(self, index_name, *key_values): + try: + index = self._indexes[index_name] + except KeyError: + raise errors.IndexDoesNotExist + doc_ids = index.lookup(key_values) + result = [] + for doc_id in doc_ids: + result.append(self._get_doc(doc_id, check_for_conflicts=True)) + return result + + def get_range_from_index(self, index_name, start_value=None, + end_value=None): + """Return all documents with key values in the specified range.""" + try: + index = self._indexes[index_name] + except KeyError: + raise errors.IndexDoesNotExist + if isinstance(start_value, basestring): + start_value = (start_value,) + if isinstance(end_value, basestring): + end_value = (end_value,) + doc_ids = index.lookup_range(start_value, end_value) + result = [] + for doc_id in doc_ids: + result.append(self._get_doc(doc_id, check_for_conflicts=True)) + return result + + def get_index_keys(self, index_name): + try: + index = self._indexes[index_name] + except KeyError: + raise errors.IndexDoesNotExist + keys = index.keys() + # XXX inefficiency warning + return list(set([tuple(key.split('\x01')) for key in keys])) + + def whats_changed(self, old_generation=0): + changes = [] + relevant_tail = self._transaction_log[old_generation:] + # We don't use len(self._transaction_log) because _transaction_log may + # get mutated by a concurrent operation. + cur_generation = old_generation + len(relevant_tail) + last_trans_id = '' + if relevant_tail: + last_trans_id = relevant_tail[-1][1] + elif self._transaction_log: + last_trans_id = self._transaction_log[-1][1] + seen = set() + generation = cur_generation + for doc_id, trans_id in reversed(relevant_tail): + if doc_id not in seen: + changes.append((doc_id, generation, trans_id)) + seen.add(doc_id) + generation -= 1 + changes.reverse() + return (cur_generation, last_trans_id, changes) + + def _force_doc_sync_conflict(self, doc): + my_doc = self._get_doc(doc.doc_id) + self._prune_conflicts(doc, vectorclock.VectorClockRev(doc.rev)) + self._conflicts.setdefault(doc.doc_id, []).append( + (my_doc.rev, my_doc.get_json())) + doc.has_conflicts = True + self._put_and_update_indexes(my_doc, doc) + + +class InMemoryIndex(object): + """Interface for managing an Index.""" + + def __init__(self, index_name, index_definition): + self._name = index_name + self._definition = index_definition + self._values = {} + parser = query_parser.Parser() + self._getters = parser.parse_all(self._definition) + + def evaluate_json(self, doc): + """Determine the 'key' after applying this index to the doc.""" + raw = json.loads(doc) + return self.evaluate(raw) + + def evaluate(self, obj): + """Evaluate a dict object, applying this definition.""" + all_rows = [[]] + for getter in self._getters: + new_rows = [] + keys = getter.get(obj) + if not keys: + return [] + for key in keys: + new_rows.extend([row + [key] for row in all_rows]) + all_rows = new_rows + all_rows = ['\x01'.join(row) for row in all_rows] + return all_rows + + def add_json(self, doc_id, doc): + """Add this json doc to the index.""" + keys = self.evaluate_json(doc) + if not keys: + return + for key in keys: + self._values.setdefault(key, []).append(doc_id) + + def remove_json(self, doc_id, doc): + """Remove this json doc from the index.""" + keys = self.evaluate_json(doc) + if keys: + for key in keys: + doc_ids = self._values[key] + doc_ids.remove(doc_id) + if not doc_ids: + del self._values[key] + + def _find_non_wildcards(self, values): + """Check if this should be a wildcard match. + + Further, this will raise an exception if the syntax is improperly + defined. + + :return: The offset of the last value we need to match against. + """ + if len(values) != len(self._definition): + raise errors.InvalidValueForIndex() + is_wildcard = False + last = 0 + for idx, val in enumerate(values): + if val.endswith('*'): + if val != '*': + # We have an 'x*' style wildcard + if is_wildcard: + # We were already in wildcard mode, so this is invalid + raise errors.InvalidGlobbing + last = idx + 1 + is_wildcard = True + else: + if is_wildcard: + # We were in wildcard mode, we can't follow that with + # non-wildcard + raise errors.InvalidGlobbing + last = idx + 1 + if not is_wildcard: + return -1 + return last + + def lookup(self, values): + """Find docs that match the values.""" + last = self._find_non_wildcards(values) + if last == -1: + return self._lookup_exact(values) + else: + return self._lookup_prefix(values[:last]) + + def lookup_range(self, start_values, end_values): + """Find docs within the range.""" + # TODO: Wildly inefficient, which is unlikely to be a problem for the + # inmemory implementation. + if start_values: + self._find_non_wildcards(start_values) + start_values = get_prefix(start_values) + if end_values: + if self._find_non_wildcards(end_values) == -1: + exact = True + else: + exact = False + end_values = get_prefix(end_values) + found = [] + for key, doc_ids in sorted(self._values.iteritems()): + if start_values and start_values > key: + continue + if end_values and end_values < key: + if exact: + break + else: + if not key.startswith(end_values): + break + found.extend(doc_ids) + return found + + def keys(self): + """Find the indexed keys.""" + return self._values.keys() + + def _lookup_prefix(self, value): + """Find docs that match the prefix string in values.""" + # TODO: We need a different data structure to make prefix style fast, + # some sort of sorted list would work, but a plain dict doesn't. + key_prefix = get_prefix(value) + all_doc_ids = [] + for key, doc_ids in sorted(self._values.iteritems()): + if key.startswith(key_prefix): + all_doc_ids.extend(doc_ids) + return all_doc_ids + + def _lookup_exact(self, value): + """Find docs that match exactly.""" + key = '\x01'.join(value) + if key in self._values: + return self._values[key] + return () + + +class InMemorySyncTarget(CommonSyncTarget): + + def get_sync_info(self, source_replica_uid): + source_gen, source_trans_id = self._db._get_replica_gen_and_trans_id( + source_replica_uid) + my_gen, my_trans_id = self._db._get_generation_info() + return ( + self._db._replica_uid, my_gen, my_trans_id, source_gen, + source_trans_id) + + def record_sync_info(self, source_replica_uid, source_replica_generation, + source_transaction_id): + if self._trace_hook: + self._trace_hook('record_sync_info') + self._db._set_replica_gen_and_trans_id( + source_replica_uid, source_replica_generation, + source_transaction_id) diff --git a/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py b/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py new file mode 100644 index 00000000..773213b5 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py @@ -0,0 +1,926 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +"""A U1DB implementation that uses SQLite as its persistence layer.""" + +import errno +import os +try: + import simplejson as json +except ImportError: + import json # noqa +from sqlite3 import dbapi2 +import sys +import time +import uuid + +import pkg_resources + +from u1db.backends import CommonBackend, CommonSyncTarget +from u1db import ( + Document, + errors, + query_parser, + vectorclock, + ) + + +class SQLiteDatabase(CommonBackend): + """A U1DB implementation that uses SQLite as its persistence layer.""" + + _sqlite_registry = {} + + def __init__(self, sqlite_file, document_factory=None): + """Create a new sqlite file.""" + self._db_handle = dbapi2.connect(sqlite_file) + self._real_replica_uid = None + self._ensure_schema() + self._factory = document_factory or Document + + def set_document_factory(self, factory): + self._factory = factory + + def get_sync_target(self): + return SQLiteSyncTarget(self) + + @classmethod + def _which_index_storage(cls, c): + try: + c.execute("SELECT value FROM u1db_config" + " WHERE name = 'index_storage'") + except dbapi2.OperationalError, e: + # The table does not exist yet + return None, e + else: + return c.fetchone()[0], None + + WAIT_FOR_PARALLEL_INIT_HALF_INTERVAL = 0.5 + + @classmethod + def _open_database(cls, sqlite_file, document_factory=None): + if not os.path.isfile(sqlite_file): + raise errors.DatabaseDoesNotExist() + tries = 2 + while True: + # Note: There seems to be a bug in sqlite 3.5.9 (with python2.6) + # where without re-opening the database on Windows, it + # doesn't see the transaction that was just committed + db_handle = dbapi2.connect(sqlite_file) + c = db_handle.cursor() + v, err = cls._which_index_storage(c) + db_handle.close() + if v is not None: + break + # possibly another process is initializing it, wait for it to be + # done + if tries == 0: + raise err # go for the richest error? + tries -= 1 + time.sleep(cls.WAIT_FOR_PARALLEL_INIT_HALF_INTERVAL) + return SQLiteDatabase._sqlite_registry[v]( + sqlite_file, document_factory=document_factory) + + @classmethod + def open_database(cls, sqlite_file, create, backend_cls=None, + document_factory=None): + try: + return cls._open_database( + sqlite_file, document_factory=document_factory) + except errors.DatabaseDoesNotExist: + if not create: + raise + if backend_cls is None: + # default is SQLitePartialExpandDatabase + backend_cls = SQLitePartialExpandDatabase + return backend_cls(sqlite_file, document_factory=document_factory) + + @staticmethod + def delete_database(sqlite_file): + try: + os.unlink(sqlite_file) + except OSError as ex: + if ex.errno == errno.ENOENT: + raise errors.DatabaseDoesNotExist() + raise + + @staticmethod + def register_implementation(klass): + """Register that we implement an SQLiteDatabase. + + The attribute _index_storage_value will be used as the lookup key. + """ + SQLiteDatabase._sqlite_registry[klass._index_storage_value] = klass + + def _get_sqlite_handle(self): + """Get access to the underlying sqlite database. + + This should only be used by the test suite, etc, for examining the + state of the underlying database. + """ + return self._db_handle + + def _close_sqlite_handle(self): + """Release access to the underlying sqlite database.""" + self._db_handle.close() + + def close(self): + self._close_sqlite_handle() + + def _is_initialized(self, c): + """Check if this database has been initialized.""" + c.execute("PRAGMA case_sensitive_like=ON") + try: + c.execute("SELECT value FROM u1db_config" + " WHERE name = 'sql_schema'") + except dbapi2.OperationalError: + # The table does not exist yet + val = None + else: + val = c.fetchone() + if val is not None: + return True + return False + + def _initialize(self, c): + """Create the schema in the database.""" + #read the script with sql commands + # TODO: Change how we set up the dependency. Most likely use something + # like lp:dirspec to grab the file from a common resource + # directory. Doesn't specifically need to be handled until we get + # to the point of packaging this. + schema_content = pkg_resources.resource_string( + __name__, 'dbschema.sql') + # Note: We'd like to use c.executescript() here, but it seems that + # executescript always commits, even if you set + # isolation_level = None, so if we want to properly handle + # exclusive locking and rollbacks between processes, we need + # to execute it line-by-line + for line in schema_content.split(';'): + if not line: + continue + c.execute(line) + #add extra fields + self._extra_schema_init(c) + # A unique identifier should be set for this replica. Implementations + # don't have to strictly use uuid here, but we do want the uid to be + # unique amongst all databases that will sync with each other. + # We might extend this to using something with hostname for easier + # debugging. + self._set_replica_uid_in_transaction(uuid.uuid4().hex) + c.execute("INSERT INTO u1db_config VALUES" " ('index_storage', ?)", + (self._index_storage_value,)) + + def _ensure_schema(self): + """Ensure that the database schema has been created.""" + old_isolation_level = self._db_handle.isolation_level + c = self._db_handle.cursor() + if self._is_initialized(c): + return + try: + # autocommit/own mgmt of transactions + self._db_handle.isolation_level = None + with self._db_handle: + # only one execution path should initialize the db + c.execute("begin exclusive") + if self._is_initialized(c): + return + self._initialize(c) + finally: + self._db_handle.isolation_level = old_isolation_level + + def _extra_schema_init(self, c): + """Add any extra fields, etc to the basic table definitions.""" + + def _parse_index_definition(self, index_field): + """Parse a field definition for an index, returning a Getter.""" + # Note: We may want to keep a Parser object around, and cache the + # Getter objects for a greater length of time. Specifically, if + # you create a bunch of indexes, and then insert 50k docs, you'll + # re-parse the indexes between puts. The time to insert the docs + # is still likely to dominate put_doc time, though. + parser = query_parser.Parser() + getter = parser.parse(index_field) + return getter + + def _update_indexes(self, doc_id, raw_doc, getters, db_cursor): + """Update document_fields for a single document. + + :param doc_id: Identifier for this document + :param raw_doc: The python dict representation of the document. + :param getters: A list of [(field_name, Getter)]. Getter.get will be + called to evaluate the index definition for this document, and the + results will be inserted into the db. + :param db_cursor: An sqlite Cursor. + :return: None + """ + values = [] + for field_name, getter in getters: + for idx_value in getter.get(raw_doc): + values.append((doc_id, field_name, idx_value)) + if values: + db_cursor.executemany( + "INSERT INTO document_fields VALUES (?, ?, ?)", values) + + def _set_replica_uid(self, replica_uid): + """Force the replica_uid to be set.""" + with self._db_handle: + self._set_replica_uid_in_transaction(replica_uid) + + def _set_replica_uid_in_transaction(self, replica_uid): + """Set the replica_uid. A transaction should already be held.""" + c = self._db_handle.cursor() + c.execute("INSERT OR REPLACE INTO u1db_config" + " VALUES ('replica_uid', ?)", + (replica_uid,)) + self._real_replica_uid = replica_uid + + def _get_replica_uid(self): + if self._real_replica_uid is not None: + return self._real_replica_uid + c = self._db_handle.cursor() + c.execute("SELECT value FROM u1db_config WHERE name = 'replica_uid'") + val = c.fetchone() + if val is None: + return None + self._real_replica_uid = val[0] + return self._real_replica_uid + + _replica_uid = property(_get_replica_uid) + + def _get_generation(self): + c = self._db_handle.cursor() + c.execute('SELECT max(generation) FROM transaction_log') + val = c.fetchone()[0] + if val is None: + return 0 + return val + + def _get_generation_info(self): + c = self._db_handle.cursor() + c.execute( + 'SELECT max(generation), transaction_id FROM transaction_log ') + val = c.fetchone() + if val[0] is None: + return(0, '') + return val + + def _get_trans_id_for_gen(self, generation): + if generation == 0: + return '' + c = self._db_handle.cursor() + c.execute( + 'SELECT transaction_id FROM transaction_log WHERE generation = ?', + (generation,)) + val = c.fetchone() + if val is None: + raise errors.InvalidGeneration + return val[0] + + def _get_transaction_log(self): + c = self._db_handle.cursor() + c.execute("SELECT doc_id, transaction_id FROM transaction_log" + " ORDER BY generation") + return c.fetchall() + + def _get_doc(self, doc_id, check_for_conflicts=False): + """Get just the document content, without fancy handling.""" + c = self._db_handle.cursor() + if check_for_conflicts: + c.execute( + "SELECT document.doc_rev, document.content, " + "count(conflicts.doc_rev) FROM document LEFT OUTER JOIN " + "conflicts ON conflicts.doc_id = document.doc_id WHERE " + "document.doc_id = ? GROUP BY document.doc_id, " + "document.doc_rev, document.content;", (doc_id,)) + else: + c.execute( + "SELECT doc_rev, content, 0 FROM document WHERE doc_id = ?", + (doc_id,)) + val = c.fetchone() + if val is None: + return None + doc_rev, content, conflicts = val + doc = self._factory(doc_id, doc_rev, content) + doc.has_conflicts = conflicts > 0 + return doc + + def _has_conflicts(self, doc_id): + c = self._db_handle.cursor() + c.execute("SELECT 1 FROM conflicts WHERE doc_id = ? LIMIT 1", + (doc_id,)) + val = c.fetchone() + if val is None: + return False + else: + return True + + def get_doc(self, doc_id, include_deleted=False): + doc = self._get_doc(doc_id, check_for_conflicts=True) + if doc is None: + return None + if doc.is_tombstone() and not include_deleted: + return None + return doc + + def get_all_docs(self, include_deleted=False): + """Get all documents from the database.""" + generation = self._get_generation() + results = [] + c = self._db_handle.cursor() + c.execute( + "SELECT document.doc_id, document.doc_rev, document.content, " + "count(conflicts.doc_rev) FROM document LEFT OUTER JOIN conflicts " + "ON conflicts.doc_id = document.doc_id GROUP BY document.doc_id, " + "document.doc_rev, document.content;") + rows = c.fetchall() + for doc_id, doc_rev, content, conflicts in rows: + if content is None and not include_deleted: + continue + doc = self._factory(doc_id, doc_rev, content) + doc.has_conflicts = conflicts > 0 + results.append(doc) + return (generation, results) + + def put_doc(self, doc): + if doc.doc_id is None: + raise errors.InvalidDocId() + self._check_doc_id(doc.doc_id) + self._check_doc_size(doc) + with self._db_handle: + old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True) + if old_doc and old_doc.has_conflicts: + raise errors.ConflictedDoc() + if old_doc and doc.rev is None and old_doc.is_tombstone(): + new_rev = self._allocate_doc_rev(old_doc.rev) + else: + if old_doc is not None: + if old_doc.rev != doc.rev: + raise errors.RevisionConflict() + else: + if doc.rev is not None: + raise errors.RevisionConflict() + new_rev = self._allocate_doc_rev(doc.rev) + doc.rev = new_rev + self._put_and_update_indexes(old_doc, doc) + return new_rev + + def _expand_to_fields(self, doc_id, base_field, raw_doc, save_none): + """Convert a dict representation into named fields. + + So something like: {'key1': 'val1', 'key2': 'val2'} + gets converted into: [(doc_id, 'key1', 'val1', 0) + (doc_id, 'key2', 'val2', 0)] + :param doc_id: Just added to every record. + :param base_field: if set, these are nested keys, so each field should + be appropriately prefixed. + :param raw_doc: The python dictionary. + """ + # TODO: Handle lists + values = [] + for field_name, value in raw_doc.iteritems(): + if value is None and not save_none: + continue + if base_field: + full_name = base_field + '.' + field_name + else: + full_name = field_name + if value is None or isinstance(value, (int, float, basestring)): + values.append((doc_id, full_name, value, len(values))) + else: + subvalues = self._expand_to_fields(doc_id, full_name, value, + save_none) + for _, subfield_name, val, _ in subvalues: + values.append((doc_id, subfield_name, val, len(values))) + return values + + def _put_and_update_indexes(self, old_doc, doc): + """Actually insert a document into the database. + + This both updates the existing documents content, and any indexes that + refer to this document. + """ + raise NotImplementedError(self._put_and_update_indexes) + + def whats_changed(self, old_generation=0): + c = self._db_handle.cursor() + c.execute("SELECT generation, doc_id, transaction_id" + " FROM transaction_log" + " WHERE generation > ? ORDER BY generation DESC", + (old_generation,)) + results = c.fetchall() + cur_gen = old_generation + seen = set() + changes = [] + newest_trans_id = '' + for generation, doc_id, trans_id in results: + if doc_id not in seen: + changes.append((doc_id, generation, trans_id)) + seen.add(doc_id) + if changes: + cur_gen = changes[0][1] # max generation + newest_trans_id = changes[0][2] + changes.reverse() + else: + c.execute("SELECT generation, transaction_id" + " FROM transaction_log ORDER BY generation DESC LIMIT 1") + results = c.fetchone() + if not results: + cur_gen = 0 + newest_trans_id = '' + else: + cur_gen, newest_trans_id = results + + return cur_gen, newest_trans_id, changes + + def delete_doc(self, doc): + with self._db_handle: + old_doc = self._get_doc(doc.doc_id, check_for_conflicts=True) + if old_doc is None: + raise errors.DocumentDoesNotExist + if old_doc.rev != doc.rev: + raise errors.RevisionConflict() + if old_doc.is_tombstone(): + raise errors.DocumentAlreadyDeleted + if old_doc.has_conflicts: + raise errors.ConflictedDoc() + new_rev = self._allocate_doc_rev(doc.rev) + doc.rev = new_rev + doc.make_tombstone() + self._put_and_update_indexes(old_doc, doc) + return new_rev + + def _get_conflicts(self, doc_id): + c = self._db_handle.cursor() + c.execute("SELECT doc_rev, content FROM conflicts WHERE doc_id = ?", + (doc_id,)) + return [self._factory(doc_id, doc_rev, content) + for doc_rev, content in c.fetchall()] + + def get_doc_conflicts(self, doc_id): + with self._db_handle: + conflict_docs = self._get_conflicts(doc_id) + if not conflict_docs: + return [] + this_doc = self._get_doc(doc_id) + this_doc.has_conflicts = True + return [this_doc] + conflict_docs + + def _get_replica_gen_and_trans_id(self, other_replica_uid): + c = self._db_handle.cursor() + c.execute("SELECT known_generation, known_transaction_id FROM sync_log" + " WHERE replica_uid = ?", + (other_replica_uid,)) + val = c.fetchone() + if val is None: + other_gen = 0 + trans_id = '' + else: + other_gen = val[0] + trans_id = val[1] + return other_gen, trans_id + + def _set_replica_gen_and_trans_id(self, other_replica_uid, + other_generation, other_transaction_id): + with self._db_handle: + self._do_set_replica_gen_and_trans_id( + other_replica_uid, other_generation, other_transaction_id) + + def _do_set_replica_gen_and_trans_id(self, other_replica_uid, + other_generation, + other_transaction_id): + c = self._db_handle.cursor() + c.execute("INSERT OR REPLACE INTO sync_log VALUES (?, ?, ?)", + (other_replica_uid, other_generation, + other_transaction_id)) + + def _put_doc_if_newer(self, doc, save_conflict, replica_uid=None, + replica_gen=None, replica_trans_id=None): + with self._db_handle: + return super(SQLiteDatabase, self)._put_doc_if_newer(doc, + save_conflict=save_conflict, + replica_uid=replica_uid, replica_gen=replica_gen, + replica_trans_id=replica_trans_id) + + def _add_conflict(self, c, doc_id, my_doc_rev, my_content): + c.execute("INSERT INTO conflicts VALUES (?, ?, ?)", + (doc_id, my_doc_rev, my_content)) + + def _delete_conflicts(self, c, doc, conflict_revs): + deleting = [(doc.doc_id, c_rev) for c_rev in conflict_revs] + c.executemany("DELETE FROM conflicts" + " WHERE doc_id=? AND doc_rev=?", deleting) + doc.has_conflicts = self._has_conflicts(doc.doc_id) + + def _prune_conflicts(self, doc, doc_vcr): + if self._has_conflicts(doc.doc_id): + autoresolved = False + c_revs_to_prune = [] + for c_doc in self._get_conflicts(doc.doc_id): + c_vcr = vectorclock.VectorClockRev(c_doc.rev) + if doc_vcr.is_newer(c_vcr): + c_revs_to_prune.append(c_doc.rev) + elif doc.same_content_as(c_doc): + c_revs_to_prune.append(c_doc.rev) + doc_vcr.maximize(c_vcr) + autoresolved = True + if autoresolved: + doc_vcr.increment(self._replica_uid) + doc.rev = doc_vcr.as_str() + c = self._db_handle.cursor() + self._delete_conflicts(c, doc, c_revs_to_prune) + + def _force_doc_sync_conflict(self, doc): + my_doc = self._get_doc(doc.doc_id) + c = self._db_handle.cursor() + self._prune_conflicts(doc, vectorclock.VectorClockRev(doc.rev)) + self._add_conflict(c, doc.doc_id, my_doc.rev, my_doc.get_json()) + doc.has_conflicts = True + self._put_and_update_indexes(my_doc, doc) + + def resolve_doc(self, doc, conflicted_doc_revs): + with self._db_handle: + cur_doc = self._get_doc(doc.doc_id) + # TODO: https://bugs.launchpad.net/u1db/+bug/928274 + # I think we have a logic bug in resolve_doc + # Specifically, cur_doc.rev is always in the final vector + # clock of revisions that we supersede, even if it wasn't in + # conflicted_doc_revs. We still add it as a conflict, but the + # fact that _put_doc_if_newer propagates resolutions means I + # think that conflict could accidentally be resolved. We need + # to add a test for this case first. (create a rev, create a + # conflict, create another conflict, resolve the first rev + # and first conflict, then make sure that the resolved + # rev doesn't supersede the second conflict rev.) It *might* + # not matter, because the superseding rev is in as a + # conflict, but it does seem incorrect + new_rev = self._ensure_maximal_rev(cur_doc.rev, + conflicted_doc_revs) + superseded_revs = set(conflicted_doc_revs) + c = self._db_handle.cursor() + doc.rev = new_rev + if cur_doc.rev in superseded_revs: + self._put_and_update_indexes(cur_doc, doc) + else: + self._add_conflict(c, doc.doc_id, new_rev, doc.get_json()) + # TODO: Is there some way that we could construct a rev that would + # end up in superseded_revs, such that we add a conflict, and + # then immediately delete it? + self._delete_conflicts(c, doc, superseded_revs) + + def list_indexes(self): + """Return the list of indexes and their definitions.""" + c = self._db_handle.cursor() + # TODO: How do we test the ordering? + c.execute("SELECT name, field FROM index_definitions" + " ORDER BY name, offset") + definitions = [] + cur_name = None + for name, field in c.fetchall(): + if cur_name != name: + definitions.append((name, [])) + cur_name = name + definitions[-1][-1].append(field) + return definitions + + def _get_index_definition(self, index_name): + """Return the stored definition for a given index_name.""" + c = self._db_handle.cursor() + c.execute("SELECT field FROM index_definitions" + " WHERE name = ? ORDER BY offset", (index_name,)) + fields = [x[0] for x in c.fetchall()] + if not fields: + raise errors.IndexDoesNotExist + return fields + + @staticmethod + def _strip_glob(value): + """Remove the trailing * from a value.""" + assert value[-1] == '*' + return value[:-1] + + def _format_query(self, definition, key_values): + # First, build the definition. We join the document_fields table + # against itself, as many times as the 'width' of our definition. + # We then do a query for each key_value, one-at-a-time. + # Note: All of these strings are static, we could cache them, etc. + tables = ["document_fields d%d" % i for i in range(len(definition))] + novalue_where = ["d.doc_id = d%d.doc_id" + " AND d%d.field_name = ?" + % (i, i) for i in range(len(definition))] + wildcard_where = [novalue_where[i] + + (" AND d%d.value NOT NULL" % (i,)) + for i in range(len(definition))] + exact_where = [novalue_where[i] + + (" AND d%d.value = ?" % (i,)) + for i in range(len(definition))] + like_where = [novalue_where[i] + + (" AND d%d.value GLOB ?" % (i,)) + for i in range(len(definition))] + is_wildcard = False + # Merge the lists together, so that: + # [field1, field2, field3], [val1, val2, val3] + # Becomes: + # (field1, val1, field2, val2, field3, val3) + args = [] + where = [] + for idx, (field, value) in enumerate(zip(definition, key_values)): + args.append(field) + if value.endswith('*'): + if value == '*': + where.append(wildcard_where[idx]) + else: + # This is a glob match + if is_wildcard: + # We can't have a partial wildcard following + # another wildcard + raise errors.InvalidGlobbing + where.append(like_where[idx]) + args.append(value) + is_wildcard = True + else: + if is_wildcard: + raise errors.InvalidGlobbing + where.append(exact_where[idx]) + args.append(value) + statement = ( + "SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM " + "document d, %s LEFT OUTER JOIN conflicts c ON c.doc_id = " + "d.doc_id WHERE %s GROUP BY d.doc_id, d.doc_rev, d.content ORDER " + "BY %s;" % (', '.join(tables), ' AND '.join(where), ', '.join( + ['d%d.value' % i for i in range(len(definition))]))) + return statement, args + + def get_from_index(self, index_name, *key_values): + definition = self._get_index_definition(index_name) + if len(key_values) != len(definition): + raise errors.InvalidValueForIndex() + statement, args = self._format_query(definition, key_values) + c = self._db_handle.cursor() + try: + c.execute(statement, tuple(args)) + except dbapi2.OperationalError, e: + raise dbapi2.OperationalError(str(e) + + '\nstatement: %s\nargs: %s\n' % (statement, args)) + res = c.fetchall() + results = [] + for row in res: + doc = self._factory(row[0], row[1], row[2]) + doc.has_conflicts = row[3] > 0 + results.append(doc) + return results + + def _format_range_query(self, definition, start_value, end_value): + tables = ["document_fields d%d" % i for i in range(len(definition))] + novalue_where = [ + "d.doc_id = d%d.doc_id AND d%d.field_name = ?" % (i, i) for i in + range(len(definition))] + wildcard_where = [ + novalue_where[i] + (" AND d%d.value NOT NULL" % (i,)) for i in + range(len(definition))] + like_where = [ + novalue_where[i] + ( + " AND (d%d.value < ? OR d%d.value GLOB ?)" % (i, i)) for i in + range(len(definition))] + range_where_lower = [ + novalue_where[i] + (" AND d%d.value >= ?" % (i,)) for i in + range(len(definition))] + range_where_upper = [ + novalue_where[i] + (" AND d%d.value <= ?" % (i,)) for i in + range(len(definition))] + args = [] + where = [] + if start_value: + if isinstance(start_value, basestring): + start_value = (start_value,) + if len(start_value) != len(definition): + raise errors.InvalidValueForIndex() + is_wildcard = False + for idx, (field, value) in enumerate(zip(definition, start_value)): + args.append(field) + if value.endswith('*'): + if value == '*': + where.append(wildcard_where[idx]) + else: + # This is a glob match + if is_wildcard: + # We can't have a partial wildcard following + # another wildcard + raise errors.InvalidGlobbing + where.append(range_where_lower[idx]) + args.append(self._strip_glob(value)) + is_wildcard = True + else: + if is_wildcard: + raise errors.InvalidGlobbing + where.append(range_where_lower[idx]) + args.append(value) + if end_value: + if isinstance(end_value, basestring): + end_value = (end_value,) + if len(end_value) != len(definition): + raise errors.InvalidValueForIndex() + is_wildcard = False + for idx, (field, value) in enumerate(zip(definition, end_value)): + args.append(field) + if value.endswith('*'): + if value == '*': + where.append(wildcard_where[idx]) + else: + # This is a glob match + if is_wildcard: + # We can't have a partial wildcard following + # another wildcard + raise errors.InvalidGlobbing + where.append(like_where[idx]) + args.append(self._strip_glob(value)) + args.append(value) + is_wildcard = True + else: + if is_wildcard: + raise errors.InvalidGlobbing + where.append(range_where_upper[idx]) + args.append(value) + statement = ( + "SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM " + "document d, %s LEFT OUTER JOIN conflicts c ON c.doc_id = " + "d.doc_id WHERE %s GROUP BY d.doc_id, d.doc_rev, d.content ORDER " + "BY %s;" % (', '.join(tables), ' AND '.join(where), ', '.join( + ['d%d.value' % i for i in range(len(definition))]))) + return statement, args + + def get_range_from_index(self, index_name, start_value=None, + end_value=None): + """Return all documents with key values in the specified range.""" + definition = self._get_index_definition(index_name) + statement, args = self._format_range_query( + definition, start_value, end_value) + c = self._db_handle.cursor() + try: + c.execute(statement, tuple(args)) + except dbapi2.OperationalError, e: + raise dbapi2.OperationalError(str(e) + + '\nstatement: %s\nargs: %s\n' % (statement, args)) + res = c.fetchall() + results = [] + for row in res: + doc = self._factory(row[0], row[1], row[2]) + doc.has_conflicts = row[3] > 0 + results.append(doc) + return results + + def get_index_keys(self, index_name): + c = self._db_handle.cursor() + definition = self._get_index_definition(index_name) + value_fields = ', '.join([ + 'd%d.value' % i for i in range(len(definition))]) + tables = ["document_fields d%d" % i for i in range(len(definition))] + novalue_where = [ + "d.doc_id = d%d.doc_id AND d%d.field_name = ?" % (i, i) for i in + range(len(definition))] + where = [ + novalue_where[i] + (" AND d%d.value NOT NULL" % (i,)) for i in + range(len(definition))] + statement = ( + "SELECT %s FROM document d, %s WHERE %s GROUP BY %s;" % ( + value_fields, ', '.join(tables), ' AND '.join(where), + value_fields)) + try: + c.execute(statement, tuple(definition)) + except dbapi2.OperationalError, e: + raise dbapi2.OperationalError(str(e) + + '\nstatement: %s\nargs: %s\n' % (statement, tuple(definition))) + return c.fetchall() + + def delete_index(self, index_name): + with self._db_handle: + c = self._db_handle.cursor() + c.execute("DELETE FROM index_definitions WHERE name = ?", + (index_name,)) + c.execute( + "DELETE FROM document_fields WHERE document_fields.field_name " + " NOT IN (SELECT field from index_definitions)") + + +class SQLiteSyncTarget(CommonSyncTarget): + + def get_sync_info(self, source_replica_uid): + source_gen, source_trans_id = self._db._get_replica_gen_and_trans_id( + source_replica_uid) + my_gen, my_trans_id = self._db._get_generation_info() + return ( + self._db._replica_uid, my_gen, my_trans_id, source_gen, + source_trans_id) + + def record_sync_info(self, source_replica_uid, source_replica_generation, + source_replica_transaction_id): + if self._trace_hook: + self._trace_hook('record_sync_info') + self._db._set_replica_gen_and_trans_id( + source_replica_uid, source_replica_generation, + source_replica_transaction_id) + + +class SQLitePartialExpandDatabase(SQLiteDatabase): + """An SQLite Backend that expands documents into a document_field table. + + It stores the original document text in document.doc. For fields that are + indexed, the data goes into document_fields. + """ + + _index_storage_value = 'expand referenced' + + def _get_indexed_fields(self): + """Determine what fields are indexed.""" + c = self._db_handle.cursor() + c.execute("SELECT field FROM index_definitions") + return set([x[0] for x in c.fetchall()]) + + def _evaluate_index(self, raw_doc, field): + parser = query_parser.Parser() + getter = parser.parse(field) + return getter.get(raw_doc) + + def _put_and_update_indexes(self, old_doc, doc): + c = self._db_handle.cursor() + if doc and not doc.is_tombstone(): + raw_doc = json.loads(doc.get_json()) + else: + raw_doc = {} + if old_doc is not None: + c.execute("UPDATE document SET doc_rev=?, content=?" + " WHERE doc_id = ?", + (doc.rev, doc.get_json(), doc.doc_id)) + c.execute("DELETE FROM document_fields WHERE doc_id = ?", + (doc.doc_id,)) + else: + c.execute("INSERT INTO document (doc_id, doc_rev, content)" + " VALUES (?, ?, ?)", + (doc.doc_id, doc.rev, doc.get_json())) + indexed_fields = self._get_indexed_fields() + if indexed_fields: + # It is expected that len(indexed_fields) is shorter than + # len(raw_doc) + getters = [(field, self._parse_index_definition(field)) + for field in indexed_fields] + self._update_indexes(doc.doc_id, raw_doc, getters, c) + trans_id = self._allocate_transaction_id() + c.execute("INSERT INTO transaction_log(doc_id, transaction_id)" + " VALUES (?, ?)", (doc.doc_id, trans_id)) + + def create_index(self, index_name, *index_expressions): + with self._db_handle: + c = self._db_handle.cursor() + cur_fields = self._get_indexed_fields() + definition = [(index_name, idx, field) + for idx, field in enumerate(index_expressions)] + try: + c.executemany("INSERT INTO index_definitions VALUES (?, ?, ?)", + definition) + except dbapi2.IntegrityError as e: + stored_def = self._get_index_definition(index_name) + if stored_def == [x[-1] for x in definition]: + return + raise errors.IndexNameTakenError, e, sys.exc_info()[2] + new_fields = set( + [f for f in index_expressions if f not in cur_fields]) + if new_fields: + self._update_all_indexes(new_fields) + + def _iter_all_docs(self): + c = self._db_handle.cursor() + c.execute("SELECT doc_id, content FROM document") + while True: + next_rows = c.fetchmany() + if not next_rows: + break + for row in next_rows: + yield row + + def _update_all_indexes(self, new_fields): + """Iterate all the documents, and add content to document_fields. + + :param new_fields: The index definitions that need to be added. + """ + getters = [(field, self._parse_index_definition(field)) + for field in new_fields] + c = self._db_handle.cursor() + for doc_id, doc in self._iter_all_docs(): + if doc is None: + continue + raw_doc = json.loads(doc) + self._update_indexes(doc_id, raw_doc, getters, c) + +SQLiteDatabase.register_implementation(SQLitePartialExpandDatabase) diff --git a/common/src/leap/soledad/common/l2db/commandline/__init__.py b/common/src/leap/soledad/common/l2db/commandline/__init__.py new file mode 100644 index 00000000..3f32e381 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/commandline/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . diff --git a/common/src/leap/soledad/common/l2db/commandline/client.py b/common/src/leap/soledad/common/l2db/commandline/client.py new file mode 100644 index 00000000..15bf8561 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/commandline/client.py @@ -0,0 +1,497 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +"""Commandline bindings for the u1db-client program.""" + +import argparse +import os +try: + import simplejson as json +except ImportError: + import json # noqa +import sys + +from u1db import ( + Document, + open as u1db_open, + sync, + errors, + ) +from u1db.commandline import command +from u1db.remote import ( + http_database, + http_target, + ) + + +client_commands = command.CommandGroup() + + +def set_oauth_credentials(client): + keys = os.environ.get('OAUTH_CREDENTIALS', None) + if keys is not None: + consumer_key, consumer_secret, \ + token_key, token_secret = keys.split(":") + client.set_oauth_credentials(consumer_key, consumer_secret, + token_key, token_secret) + + +class OneDbCmd(command.Command): + """Base class for commands operating on one local or remote database.""" + + def _open(self, database, create): + if database.startswith(('http://', 'https://')): + db = http_database.HTTPDatabase(database) + set_oauth_credentials(db) + db.open(create) + return db + else: + return u1db_open(database, create) + + +class CmdCreate(OneDbCmd): + """Create a new document from scratch""" + + name = 'create' + + @classmethod + def _populate_subparser(cls, parser): + parser.add_argument('database', + help='The local or remote database to update', + metavar='database-path-or-url') + parser.add_argument('infile', nargs='?', default=None, + help='The file to read content from.') + parser.add_argument('--id', dest='doc_id', default=None, + help='Set the document identifier') + + def run(self, database, infile, doc_id): + if infile is None: + infile = self.stdin + db = self._open(database, create=False) + doc = db.create_doc_from_json(infile.read(), doc_id=doc_id) + self.stderr.write('id: %s\nrev: %s\n' % (doc.doc_id, doc.rev)) + +client_commands.register(CmdCreate) + + +class CmdDelete(OneDbCmd): + """Delete a document from the database""" + + name = 'delete' + + @classmethod + def _populate_subparser(cls, parser): + parser.add_argument('database', + help='The local or remote database to update', + metavar='database-path-or-url') + parser.add_argument('doc_id', help='The document id to retrieve') + parser.add_argument('doc_rev', + help='The revision of the document (which is being superseded.)') + + def run(self, database, doc_id, doc_rev): + db = self._open(database, create=False) + doc = Document(doc_id, doc_rev, None) + db.delete_doc(doc) + self.stderr.write('rev: %s\n' % (doc.rev,)) + +client_commands.register(CmdDelete) + + +class CmdGet(OneDbCmd): + """Extract a document from the database""" + + name = 'get' + + @classmethod + def _populate_subparser(cls, parser): + parser.add_argument('database', + help='The local or remote database to query', + metavar='database-path-or-url') + parser.add_argument('doc_id', help='The document id to retrieve.') + parser.add_argument('outfile', nargs='?', default=None, + help='The file to write the document to', + type=argparse.FileType('wb')) + + def run(self, database, doc_id, outfile): + if outfile is None: + outfile = self.stdout + try: + db = self._open(database, create=False) + except errors.DatabaseDoesNotExist: + self.stderr.write("Database does not exist.\n") + return 1 + doc = db.get_doc(doc_id) + if doc is None: + self.stderr.write('Document not found (id: %s)\n' % (doc_id,)) + return 1 # failed + if doc.is_tombstone(): + outfile.write('[document deleted]\n') + else: + outfile.write(doc.get_json() + '\n') + self.stderr.write('rev: %s\n' % (doc.rev,)) + if doc.has_conflicts: + self.stderr.write("Document has conflicts.\n") + +client_commands.register(CmdGet) + + +class CmdGetDocConflicts(OneDbCmd): + """Get the conflicts from a document""" + + name = 'get-doc-conflicts' + + @classmethod + def _populate_subparser(cls, parser): + parser.add_argument('database', + help='The local database to query', + metavar='database-path') + parser.add_argument('doc_id', help='The document id to retrieve.') + + def run(self, database, doc_id): + try: + db = self._open(database, False) + except errors.DatabaseDoesNotExist: + self.stderr.write("Database does not exist.\n") + return 1 + conflicts = db.get_doc_conflicts(doc_id) + if not conflicts: + if db.get_doc(doc_id) is None: + self.stderr.write("Document does not exist.\n") + return 1 + self.stdout.write("[") + for i, doc in enumerate(conflicts): + if i: + self.stdout.write(",") + self.stdout.write( + json.dumps(dict(rev=doc.rev, content=doc.content), indent=4)) + self.stdout.write("]\n") + +client_commands.register(CmdGetDocConflicts) + + +class CmdInitDB(OneDbCmd): + """Create a new database""" + + name = 'init-db' + + @classmethod + def _populate_subparser(cls, parser): + parser.add_argument('database', + help='The local or remote database to create', + metavar='database-path-or-url') + parser.add_argument('--replica-uid', default=None, + help='The unique identifier for this database (not for remote)') + + def run(self, database, replica_uid): + db = self._open(database, create=True) + if replica_uid is not None: + db._set_replica_uid(replica_uid) + +client_commands.register(CmdInitDB) + + +class CmdPut(OneDbCmd): + """Add a document to the database""" + + name = 'put' + + @classmethod + def _populate_subparser(cls, parser): + parser.add_argument('database', + help='The local or remote database to update', + metavar='database-path-or-url'), + parser.add_argument('doc_id', help='The document id to retrieve') + parser.add_argument('doc_rev', + help='The revision of the document (which is being superseded.)') + parser.add_argument('infile', nargs='?', default=None, + help='The filename of the document that will be used for content', + type=argparse.FileType('rb')) + + def run(self, database, doc_id, doc_rev, infile): + if infile is None: + infile = self.stdin + try: + db = self._open(database, create=False) + doc = Document(doc_id, doc_rev, infile.read()) + doc_rev = db.put_doc(doc) + self.stderr.write('rev: %s\n' % (doc_rev,)) + except errors.DatabaseDoesNotExist: + self.stderr.write("Database does not exist.\n") + except errors.RevisionConflict: + if db.get_doc(doc_id) is None: + self.stderr.write("Document does not exist.\n") + else: + self.stderr.write("Given revision is not current.\n") + except errors.ConflictedDoc: + self.stderr.write( + "Document has conflicts.\n" + "Inspect with get-doc-conflicts, then resolve.\n") + else: + return + return 1 + +client_commands.register(CmdPut) + + +class CmdResolve(OneDbCmd): + """Resolve a conflicted document""" + + name = 'resolve-doc' + + @classmethod + def _populate_subparser(cls, parser): + parser.add_argument('database', + help='The local or remote database to update', + metavar='database-path-or-url'), + parser.add_argument('doc_id', help='The conflicted document id') + parser.add_argument('doc_revs', metavar="doc-rev", nargs="+", + help='The revisions that the new content supersedes') + parser.add_argument('--infile', nargs='?', default=None, + help='The filename of the document that will be used for content', + type=argparse.FileType('rb')) + + def run(self, database, doc_id, doc_revs, infile): + if infile is None: + infile = self.stdin + try: + db = self._open(database, create=False) + except errors.DatabaseDoesNotExist: + self.stderr.write("Database does not exist.\n") + return 1 + doc = db.get_doc(doc_id) + if doc is None: + self.stderr.write("Document does not exist.\n") + return 1 + doc.set_json(infile.read()) + db.resolve_doc(doc, doc_revs) + self.stderr.write("rev: %s\n" % db.get_doc(doc_id).rev) + if doc.has_conflicts: + self.stderr.write("Document still has conflicts.\n") + +client_commands.register(CmdResolve) + + +class CmdSync(command.Command): + """Synchronize two databases""" + + name = 'sync' + + @classmethod + def _populate_subparser(cls, parser): + parser.add_argument('source', help='database to sync from') + parser.add_argument('target', help='database to sync to') + + def _open_target(self, target): + if target.startswith(('http://', 'https://')): + st = http_target.HTTPSyncTarget.connect(target) + set_oauth_credentials(st) + else: + db = u1db_open(target, create=True) + st = db.get_sync_target() + return st + + def run(self, source, target): + """Start a Sync request.""" + source_db = u1db_open(source, create=False) + st = self._open_target(target) + syncer = sync.Synchronizer(source_db, st) + syncer.sync() + source_db.close() + +client_commands.register(CmdSync) + + +class CmdCreateIndex(OneDbCmd): + """Create an index""" + + name = "create-index" + + @classmethod + def _populate_subparser(cls, parser): + parser.add_argument('database', help='The local database to update', + metavar='database-path') + parser.add_argument('index', help='the name of the index') + parser.add_argument('expression', help='an index expression', + nargs='+') + + def run(self, database, index, expression): + try: + db = self._open(database, create=False) + db.create_index(index, *expression) + except errors.DatabaseDoesNotExist: + self.stderr.write("Database does not exist.\n") + return 1 + except errors.IndexNameTakenError: + self.stderr.write("There is already a different index named %r.\n" + % (index,)) + return 1 + except errors.IndexDefinitionParseError: + self.stderr.write("Bad index expression.\n") + return 1 + +client_commands.register(CmdCreateIndex) + + +class CmdListIndexes(OneDbCmd): + """List existing indexes""" + + name = "list-indexes" + + @classmethod + def _populate_subparser(cls, parser): + parser.add_argument('database', help='The local database to query', + metavar='database-path') + + def run(self, database): + try: + db = self._open(database, create=False) + except errors.DatabaseDoesNotExist: + self.stderr.write("Database does not exist.\n") + return 1 + for (index, expression) in db.list_indexes(): + self.stdout.write("%s: %s\n" % (index, ", ".join(expression))) + +client_commands.register(CmdListIndexes) + + +class CmdDeleteIndex(OneDbCmd): + """Delete an index""" + + name = "delete-index" + + @classmethod + def _populate_subparser(cls, parser): + parser.add_argument('database', help='The local database to update', + metavar='database-path') + parser.add_argument('index', help='the name of the index') + + def run(self, database, index): + try: + db = self._open(database, create=False) + except errors.DatabaseDoesNotExist: + self.stderr.write("Database does not exist.\n") + return 1 + db.delete_index(index) + +client_commands.register(CmdDeleteIndex) + + +class CmdGetIndexKeys(OneDbCmd): + """Get the index's keys""" + + name = "get-index-keys" + + @classmethod + def _populate_subparser(cls, parser): + parser.add_argument('database', help='The local database to query', + metavar='database-path') + parser.add_argument('index', help='the name of the index') + + def run(self, database, index): + try: + db = self._open(database, create=False) + for key in db.get_index_keys(index): + self.stdout.write("%s\n" % (", ".join( + [i.encode('utf-8') for i in key],))) + except errors.DatabaseDoesNotExist: + self.stderr.write("Database does not exist.\n") + except errors.IndexDoesNotExist: + self.stderr.write("Index does not exist.\n") + else: + return + return 1 + +client_commands.register(CmdGetIndexKeys) + + +class CmdGetFromIndex(OneDbCmd): + """Find documents by searching an index""" + + name = "get-from-index" + argv = None + + @classmethod + def _populate_subparser(cls, parser): + parser.add_argument('database', help='The local database to query', + metavar='database-path') + parser.add_argument('index', help='the name of the index') + parser.add_argument('values', metavar="value", + help='the value to look up (one per index column)', + nargs="+") + + def run(self, database, index, values): + try: + db = self._open(database, create=False) + docs = db.get_from_index(index, *values) + except errors.DatabaseDoesNotExist: + self.stderr.write("Database does not exist.\n") + except errors.IndexDoesNotExist: + self.stderr.write("Index does not exist.\n") + except errors.InvalidValueForIndex: + index_def = db._get_index_definition(index) + len_diff = len(index_def) - len(values) + if len_diff == 0: + # can't happen (HAH) + raise + argv = self.argv if self.argv is not None else sys.argv + self.stderr.write( + "Invalid query: " + "index %r requires %d query expression%s%s.\n" + "For example, the following would be valid:\n" + " %s %s %r %r %s\n" + % (index, + len(index_def), + "s" if len(index_def) > 1 else "", + ", not %d" % len(values) if len(values) else "", + argv[0], argv[1], database, index, + " ".join(map(repr, + values[:len(index_def)] + + ["*" for i in range(len_diff)])), + )) + except errors.InvalidGlobbing: + argv = self.argv if self.argv is not None else sys.argv + fixed = [] + for (i, v) in enumerate(values): + fixed.append(v) + if v.endswith('*'): + break + # values has at least one element, so i is defined + fixed.extend('*' * (len(values) - i - 1)) + self.stderr.write( + "Invalid query: a star can only be followed by stars.\n" + "For example, the following would be valid:\n" + " %s %s %r %r %s\n" + % (argv[0], argv[1], database, index, + " ".join(map(repr, fixed)))) + + else: + self.stdout.write("[") + for i, doc in enumerate(docs): + if i: + self.stdout.write(",") + self.stdout.write( + json.dumps( + dict(id=doc.doc_id, rev=doc.rev, content=doc.content), + indent=4)) + self.stdout.write("]\n") + return + return 1 + +client_commands.register(CmdGetFromIndex) + + +def main(args): + return client_commands.run_argv(args, sys.stdin, sys.stdout, sys.stderr) diff --git a/common/src/leap/soledad/common/l2db/commandline/command.py b/common/src/leap/soledad/common/l2db/commandline/command.py new file mode 100644 index 00000000..eace0560 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/commandline/command.py @@ -0,0 +1,80 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +"""Command infrastructure for u1db""" + +import argparse +import inspect + + +class CommandGroup(object): + """A collection of commands.""" + + def __init__(self, description=None): + self.commands = {} + self.description = description + + def register(self, cmd): + """Register a new command to be incorporated with this group.""" + self.commands[cmd.name] = cmd + + def make_argparser(self): + """Create an argparse.ArgumentParser""" + parser = argparse.ArgumentParser(description=self.description) + subs = parser.add_subparsers(title='commands') + for name, cmd in sorted(self.commands.iteritems()): + sub = subs.add_parser(name, help=cmd.__doc__) + sub.set_defaults(subcommand=cmd) + cmd._populate_subparser(sub) + return parser + + def run_argv(self, argv, stdin, stdout, stderr): + """Run a command, from a sys.argv[1:] style input.""" + parser = self.make_argparser() + args = parser.parse_args(argv) + cmd = args.subcommand(stdin, stdout, stderr) + params, _, _, _ = inspect.getargspec(cmd.run) + vals = [] + for param in params[1:]: + vals.append(getattr(args, param)) + return cmd.run(*vals) + + +class Command(object): + """Definition of a Command that can be run. + + :cvar name: The name of the command, so that you can run + 'u1db-client '. + """ + + name = None + + def __init__(self, stdin, stdout, stderr): + self.stdin = stdin + self.stdout = stdout + self.stderr = stderr + + @classmethod + def _populate_subparser(cls, parser): + """Child classes should override this to provide their arguments.""" + raise NotImplementedError(cls._populate_subparser) + + def run(self, *args): + """This is where the magic happens. + + Subclasses should implement this, requesting their specific arguments. + """ + raise NotImplementedError(self.run) diff --git a/common/src/leap/soledad/common/l2db/commandline/serve.py b/common/src/leap/soledad/common/l2db/commandline/serve.py new file mode 100644 index 00000000..5e10f9cb --- /dev/null +++ b/common/src/leap/soledad/common/l2db/commandline/serve.py @@ -0,0 +1,58 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +"""Build server for u1db-serve.""" +import os + +from paste import httpserver + +from u1db.remote import ( + http_app, + server_state, + cors_middleware + ) + + +class DbListingServerState(server_state.ServerState): + """ServerState capable of listing dbs.""" + + def global_info(self): + """Return list of dbs.""" + dbs = [] + for fname in os.listdir(self._workingdir): + p = os.path.join(self._workingdir, fname) + if os.path.isfile(p) and os.access(p, os.R_OK|os.W_OK): + try: + with open(p, 'rb') as f: + header = f.read(16) + if header == "SQLite format 3\000": + dbs.append(fname) + except IOError: + pass + return {"databases": dict.fromkeys(dbs), "db_count": len(dbs)} + + +def make_server(host, port, working_dir, accept_cors_connections=None): + """Make a server on host and port exposing dbs living in working_dir.""" + state = DbListingServerState() + state.set_workingdir(working_dir) + application = http_app.HTTPApp(state) + if accept_cors_connections: + application = cors_middleware.CORSMiddleware(application, + accept_cors_connections) + server = httpserver.WSGIServer(application, (host, port), + httpserver.WSGIHandler) + return server diff --git a/common/src/leap/soledad/common/l2db/errors.py b/common/src/leap/soledad/common/l2db/errors.py new file mode 100644 index 00000000..e5ee8f45 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/errors.py @@ -0,0 +1,195 @@ +# Copyright 2011-2012 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +"""A list of errors that u1db can raise.""" + + +class U1DBError(Exception): + """Generic base class for U1DB errors.""" + + # description/tag for identifying the error during transmission (http,...) + wire_description = "error" + + def __init__(self, message=None): + self.message = message + + +class RevisionConflict(U1DBError): + """The document revisions supplied does not match the current version.""" + + wire_description = "revision conflict" + + +class InvalidJSON(U1DBError): + """Content was not valid json.""" + + +class InvalidContent(U1DBError): + """Content was not a python dictionary.""" + + +class InvalidDocId(U1DBError): + """A document was requested with an invalid document identifier.""" + + wire_description = "invalid document id" + + +class MissingDocIds(U1DBError): + """Needs document ids.""" + + wire_description = "missing document ids" + + +class DocumentTooBig(U1DBError): + """Document exceeds the maximum document size for this database.""" + + wire_description = "document too big" + + +class UserQuotaExceeded(U1DBError): + """Document exceeds the maximum document size for this database.""" + + wire_description = "user quota exceeded" + + +class SubscriptionNeeded(U1DBError): + """User needs a subscription to be able to use this replica..""" + + wire_description = "user needs subscription" + + +class InvalidTransactionId(U1DBError): + """Invalid transaction for generation.""" + + wire_description = "invalid transaction id" + + +class InvalidGeneration(U1DBError): + """Generation was previously synced with a different transaction id.""" + + wire_description = "invalid generation" + + +class InvalidReplicaUID(U1DBError): + """Attempting to sync a database with itself.""" + + wire_description = "invalid replica uid" + + +class ConflictedDoc(U1DBError): + """The document is conflicted, you must call resolve before put()""" + + +class InvalidValueForIndex(U1DBError): + """The values supplied does not match the index definition.""" + + +class InvalidGlobbing(U1DBError): + """Raised if wildcard matches are not strictly at the tail of the request. + """ + + +class DocumentDoesNotExist(U1DBError): + """The document does not exist.""" + + wire_description = "document does not exist" + + +class DocumentAlreadyDeleted(U1DBError): + """The document was already deleted.""" + + wire_description = "document already deleted" + + +class DatabaseDoesNotExist(U1DBError): + """The database does not exist.""" + + wire_description = "database does not exist" + + +class IndexNameTakenError(U1DBError): + """The given index name is already taken.""" + + +class IndexDefinitionParseError(U1DBError): + """The index definition cannot be parsed.""" + + +class IndexDoesNotExist(U1DBError): + """No index of that name exists.""" + + +class Unauthorized(U1DBError): + """Request wasn't authorized properly.""" + + wire_description = "unauthorized" + + +class HTTPError(U1DBError): + """Unspecific HTTP errror.""" + + wire_description = None + + def __init__(self, status, message=None, headers={}): + self.status = status + self.message = message + self.headers = headers + + def __str__(self): + if not self.message: + return "HTTPError(%d)" % self.status + else: + return "HTTPError(%d, %r)" % (self.status, self.message) + + +class Unavailable(HTTPError): + """Server not available not serve request.""" + + wire_description = "unavailable" + + def __init__(self, message=None, headers={}): + super(Unavailable, self).__init__(503, message, headers) + + def __str__(self): + if not self.message: + return "Unavailable()" + else: + return "Unavailable(%r)" % self.message + + +class BrokenSyncStream(U1DBError): + """Unterminated or otherwise broken sync exchange stream.""" + + wire_description = None + + +class UnknownAuthMethod(U1DBError): + """Unknown auhorization method.""" + + wire_description = None + + +# mapping wire (transimission) descriptions/tags for errors to the exceptions +wire_description_to_exc = dict( + (x.wire_description, x) for x in globals().values() + if getattr(x, 'wire_description', None) not in (None, "error") +) +wire_description_to_exc["error"] = U1DBError + + +# +# wire error descriptions not corresponding to an exception +DOCUMENT_DELETED = "document deleted" diff --git a/common/src/leap/soledad/common/l2db/query_parser.py b/common/src/leap/soledad/common/l2db/query_parser.py new file mode 100644 index 00000000..f564821f --- /dev/null +++ b/common/src/leap/soledad/common/l2db/query_parser.py @@ -0,0 +1,370 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +"""Code for parsing Index definitions.""" + +import re +from u1db import ( + errors, + ) + + +class Getter(object): + """Get values from a document based on a specification.""" + + def get(self, raw_doc): + """Get a value from the document. + + :param raw_doc: a python dictionary to get the value from. + :return: A list of values that match the description. + """ + raise NotImplementedError(self.get) + + +class StaticGetter(Getter): + """A getter that returns a defined value (independent of the doc).""" + + def __init__(self, value): + """Create a StaticGetter. + + :param value: the value to return when get is called. + """ + if value is None: + self.value = [] + elif isinstance(value, list): + self.value = value + else: + self.value = [value] + + def get(self, raw_doc): + return self.value + + +def extract_field(raw_doc, subfields, index=0): + if not isinstance(raw_doc, dict): + return [] + val = raw_doc.get(subfields[index]) + if val is None: + return [] + if index < len(subfields) - 1: + if isinstance(val, list): + results = [] + for item in val: + results.extend(extract_field(item, subfields, index + 1)) + return results + if isinstance(val, dict): + return extract_field(val, subfields, index + 1) + return [] + if isinstance(val, dict): + return [] + if isinstance(val, list): + # Strip anything in the list that isn't a simple type + return [v for v in val if not isinstance(v, (dict, list))] + return [val] + + +class ExtractField(Getter): + """Extract a field from the document.""" + + def __init__(self, field): + """Create an ExtractField object. + + When a document is passed to get() this will return a value + from the document based on the field specifier passed to + the constructor. + + None will be returned if the field is nonexistant, or refers to an + object, rather than a simple type or list of simple types. + + :param field: a specifier for the field to return. + This is either a field name, or a dotted field name. + """ + self.field = field.split('.') + + def get(self, raw_doc): + return extract_field(raw_doc, self.field) + + +class Transformation(Getter): + """A transformation on a value from another Getter.""" + + name = None + arity = 1 + args = ['expression'] + + def __init__(self, inner): + """Create a transformation. + + :param inner: the argument(s) to the transformation. + """ + self.inner = inner + + def get(self, raw_doc): + inner_values = self.inner.get(raw_doc) + assert isinstance(inner_values, list),\ + 'get() should always return a list' + return self.transform(inner_values) + + def transform(self, values): + """Transform the values. + + This should be implemented by subclasses to transform the + value when get() is called. + + :param values: the values from the other Getter + :return: the transformed values. + """ + raise NotImplementedError(self.transform) + + +class Lower(Transformation): + """Lowercase a string. + + This transformation will return None for non-string inputs. However, + it will lowercase any strings in a list, dropping any elements + that are not strings. + """ + + name = "lower" + + def _can_transform(self, val): + return isinstance(val, basestring) + + def transform(self, values): + if not values: + return [] + return [val.lower() for val in values if self._can_transform(val)] + + +class Number(Transformation): + """Convert an integer to a zero padded string. + + This transformation will return None for non-integer inputs. However, it + will transform any integers in a list, dropping any elements that are not + integers. + """ + + name = 'number' + arity = 2 + args = ['expression', int] + + def __init__(self, inner, number): + super(Number, self).__init__(inner) + self.padding = "%%0%sd" % number + + def _can_transform(self, val): + return isinstance(val, int) and not isinstance(val, bool) + + def transform(self, values): + """Transform any integers in values into zero padded strings.""" + if not values: + return [] + return [self.padding % (v,) for v in values if self._can_transform(v)] + + +class Bool(Transformation): + """Convert bool to string.""" + + name = "bool" + args = ['expression'] + + def _can_transform(self, val): + return isinstance(val, bool) + + def transform(self, values): + """Transform any booleans in values into strings.""" + if not values: + return [] + return [('1' if v else '0') for v in values if self._can_transform(v)] + + +class SplitWords(Transformation): + """Split a string on whitespace. + + This Getter will return [] for non-string inputs. It will however + split any strings in an input list, discarding any elements that + are not strings. + """ + + name = "split_words" + + def _can_transform(self, val): + return isinstance(val, basestring) + + def transform(self, values): + if not values: + return [] + result = set() + for value in values: + if self._can_transform(value): + for word in value.split(): + result.add(word) + return list(result) + + +class Combine(Transformation): + """Combine multiple expressions into a single index.""" + + name = "combine" + # variable number of args + arity = -1 + + def __init__(self, *inner): + super(Combine, self).__init__(inner) + + def get(self, raw_doc): + inner_values = [] + for inner in self.inner: + inner_values.extend(inner.get(raw_doc)) + return self.transform(inner_values) + + def transform(self, values): + return values + + +class IsNull(Transformation): + """Indicate whether the input is None. + + This Getter returns a bool indicating whether the input is nil. + """ + + name = "is_null" + + def transform(self, values): + return [len(values) == 0] + + +def check_fieldname(fieldname): + if fieldname.endswith('.'): + raise errors.IndexDefinitionParseError( + "Fieldname cannot end in '.':%s^" % (fieldname,)) + + +class Parser(object): + """Parse an index expression into a sequence of transformations.""" + + _transformations = {} + _delimiters = re.compile("\(|\)|,") + + def __init__(self): + self._tokens = [] + + def _set_expression(self, expression): + self._open_parens = 0 + self._tokens = [] + expression = expression.strip() + while expression: + delimiter = self._delimiters.search(expression) + if delimiter: + idx = delimiter.start() + if idx == 0: + result, expression = (expression[:1], expression[1:]) + self._tokens.append(result) + else: + result, expression = (expression[:idx], expression[idx:]) + result = result.strip() + if result: + self._tokens.append(result) + else: + expression = expression.strip() + if expression: + self._tokens.append(expression) + expression = None + + def _get_token(self): + if self._tokens: + return self._tokens.pop(0) + + def _peek_token(self): + if self._tokens: + return self._tokens[0] + + @staticmethod + def _to_getter(term): + if isinstance(term, Getter): + return term + check_fieldname(term) + return ExtractField(term) + + def _parse_op(self, op_name): + self._get_token() # '(' + op = self._transformations.get(op_name, None) + if op is None: + raise errors.IndexDefinitionParseError( + "Unknown operation: %s" % op_name) + args = [] + while True: + args.append(self._parse_term()) + sep = self._get_token() + if sep == ')': + break + if sep != ',': + raise errors.IndexDefinitionParseError( + "Unexpected token '%s' in parentheses." % (sep,)) + parsed = [] + for i, arg in enumerate(args): + arg_type = op.args[i % len(op.args)] + if arg_type == 'expression': + inner = self._to_getter(arg) + else: + try: + inner = arg_type(arg) + except ValueError, e: + raise errors.IndexDefinitionParseError( + "Invalid value %r for argument type %r " + "(%r)." % (arg, arg_type, e)) + parsed.append(inner) + return op(*parsed) + + def _parse_term(self): + term = self._get_token() + if term is None: + raise errors.IndexDefinitionParseError( + "Unexpected end of index definition.") + if term in (',', ')', '('): + raise errors.IndexDefinitionParseError( + "Unexpected token '%s' at start of expression." % (term,)) + next_token = self._peek_token() + if next_token == '(': + return self._parse_op(term) + return term + + def parse(self, expression): + self._set_expression(expression) + term = self._to_getter(self._parse_term()) + if self._peek_token(): + raise errors.IndexDefinitionParseError( + "Unexpected token '%s' after end of expression." + % (self._peek_token(),)) + return term + + def parse_all(self, fields): + return [self.parse(field) for field in fields] + + @classmethod + def register_transormation(cls, transform): + assert transform.name not in cls._transformations, ( + "Transform %s already registered for %s" + % (transform.name, cls._transformations[transform.name])) + cls._transformations[transform.name] = transform + + +Parser.register_transormation(SplitWords) +Parser.register_transormation(Lower) +Parser.register_transormation(Number) +Parser.register_transormation(Bool) +Parser.register_transormation(IsNull) +Parser.register_transormation(Combine) diff --git a/common/src/leap/soledad/common/l2db/remote/__init__.py b/common/src/leap/soledad/common/l2db/remote/__init__.py new file mode 100644 index 00000000..3f32e381 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/remote/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . diff --git a/common/src/leap/soledad/common/l2db/remote/basic_auth_middleware.py b/common/src/leap/soledad/common/l2db/remote/basic_auth_middleware.py new file mode 100644 index 00000000..a2cbff62 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/remote/basic_auth_middleware.py @@ -0,0 +1,68 @@ +# Copyright 2012 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . +"""U1DB Basic Auth authorisation WSGI middleware.""" +import httplib +try: + import simplejson as json +except ImportError: + import json # noqa +from wsgiref.util import shift_path_info + + +class Unauthorized(Exception): + """User authorization failed.""" + + +class BasicAuthMiddleware(object): + """U1DB Basic Auth Authorisation WSGI middleware.""" + + def __init__(self, app, prefix): + self.app = app + self.prefix = prefix + + def _error(self, start_response, status, description, message=None): + start_response("%d %s" % (status, httplib.responses[status]), + [('content-type', 'application/json')]) + err = {"error": description} + if message: + err['message'] = message + return [json.dumps(err)] + + def __call__(self, environ, start_response): + if self.prefix and not environ['PATH_INFO'].startswith(self.prefix): + return self._error(start_response, 400, "bad request") + auth = environ.get('HTTP_AUTHORIZATION') + if not auth: + return self._error(start_response, 401, "unauthorized", + "Missing Basic Authentication.") + scheme, encoded = auth.split(None, 1) + if scheme.lower() != 'basic': + return self._error( + start_response, 401, "unauthorized", + "Missing Basic Authentication") + user, password = encoded.decode('base64').split(':', 1) + try: + self.verify_user(environ, user, password) + except Unauthorized: + return self._error( + start_response, 401, "unauthorized", + "Incorrect password or login.") + del environ['HTTP_AUTHORIZATION'] + shift_path_info(environ) + return self.app(environ, start_response) + + def verify_user(self, environ, username, password): + raise NotImplementedError(self.verify_user) diff --git a/common/src/leap/soledad/common/l2db/remote/cors_middleware.py b/common/src/leap/soledad/common/l2db/remote/cors_middleware.py new file mode 100644 index 00000000..8041b968 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/remote/cors_middleware.py @@ -0,0 +1,42 @@ +# Copyright 2012 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . +"""U1DB Cross-Origin Resource Sharing WSGI middleware.""" + + +class CORSMiddleware(object): + """U1DB Cross-Origin Resource Sharing WSGI middleware.""" + + def __init__(self, app, accept_cors_connections): + self.origins = ' '.join(accept_cors_connections) + self.app = app + + def _cors_headers(self): + return [('access-control-allow-origin', self.origins), + ('access-control-allow-headers', + 'authorization, content-type, x-requested-with'), + ('access-control-allow-methods', + 'GET, POST, PUT, DELETE, OPTIONS')] + + def __call__(self, environ, start_response): + def wrap_start_response(status, headers, exc_info=None): + headers += self._cors_headers() + return start_response(status, headers, exc_info) + + if environ['REQUEST_METHOD'].lower() == 'options': + wrap_start_response("200 OK", [('content-type', 'text/plain')]) + return [''] + + return self.app(environ, wrap_start_response) diff --git a/common/src/leap/soledad/common/l2db/remote/http_app.py b/common/src/leap/soledad/common/l2db/remote/http_app.py new file mode 100644 index 00000000..85cdb029 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/remote/http_app.py @@ -0,0 +1,661 @@ +# Copyright 2011-2012 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +"""HTTP Application exposing U1DB.""" + +import functools +import httplib +import inspect +try: + import simplejson as json +except ImportError: + import json # noqa +import sys +import urlparse + +import routes.mapper + +from u1db import ( + __version__ as _u1db_version, + DBNAME_CONSTRAINTS, + Document, + errors, + sync, + ) +from u1db.remote import ( + http_errors, + utils, + ) + + +def parse_bool(expression): + """Parse boolean querystring parameter.""" + if expression == 'true': + return True + return False + + +def parse_list(expression): + if not expression: + return [] + return [t.strip() for t in expression.split(',')] + + +def none_or_str(expression): + if expression is None: + return None + return str(expression) + + +class BadRequest(Exception): + """Bad request.""" + + +class _FencedReader(object): + """Read and get lines from a file but not past a given length.""" + + MAXCHUNK = 8192 + + def __init__(self, rfile, total, max_entry_size): + self.rfile = rfile + self.remaining = total + self.max_entry_size = max_entry_size + self._kept = None + + def read_chunk(self, atmost): + if self._kept is not None: + # ignore atmost, kept data should be a subchunk anyway + kept, self._kept = self._kept, None + return kept + if self.remaining == 0: + return '' + data = self.rfile.read(min(self.remaining, atmost)) + self.remaining -= len(data) + return data + + def getline(self): + line_parts = [] + size = 0 + while True: + chunk = self.read_chunk(self.MAXCHUNK) + if chunk == '': + break + nl = chunk.find("\n") + if nl != -1: + size += nl + 1 + if size > self.max_entry_size: + raise BadRequest + line_parts.append(chunk[:nl + 1]) + rest = chunk[nl + 1:] + self._kept = rest or None + break + else: + size += len(chunk) + if size > self.max_entry_size: + raise BadRequest + line_parts.append(chunk) + return ''.join(line_parts) + + +def http_method(**control): + """Decoration for handling of query arguments and content for a HTTP + method. + + args and content here are the query arguments and body of the incoming + HTTP requests. + + Match query arguments to python method arguments: + w = http_method()(f) + w(self, args, content) => args["content"]=content; + f(self, **args) + + JSON deserialize content to arguments: + w = http_method(content_as_args=True,...)(f) + w(self, args, content) => args.update(json.loads(content)); + f(self, **args) + + Support conversions (e.g int): + w = http_method(Arg=Conv,...)(f) + w(self, args, content) => args["Arg"]=Conv(args["Arg"]); + f(self, **args) + + Enforce no use of query arguments: + w = http_method(no_query=True,...)(f) + w(self, args, content) raises BadRequest if args is not empty + + Argument mismatches, deserialisation failures produce BadRequest. + """ + content_as_args = control.pop('content_as_args', False) + no_query = control.pop('no_query', False) + conversions = control.items() + + def wrap(f): + argspec = inspect.getargspec(f) + assert argspec.args[0] == "self" + nargs = len(argspec.args) + ndefaults = len(argspec.defaults or ()) + required_args = set(argspec.args[1:nargs - ndefaults]) + all_args = set(argspec.args) + + @functools.wraps(f) + def wrapper(self, args, content): + if no_query and args: + raise BadRequest() + if content is not None: + if content_as_args: + try: + args.update(json.loads(content)) + except ValueError: + raise BadRequest() + else: + args["content"] = content + if not (required_args <= set(args) <= all_args): + raise BadRequest("Missing required arguments.") + for name, conv in conversions: + if name not in args: + continue + try: + args[name] = conv(args[name]) + except ValueError: + raise BadRequest() + return f(self, **args) + + return wrapper + + return wrap + + +class URLToResource(object): + """Mappings from URLs to resources.""" + + def __init__(self): + self._map = routes.mapper.Mapper(controller_scan=None) + + def register(self, resource_cls): + # register + self._map.connect(None, resource_cls.url_pattern, + resource_cls=resource_cls, + requirements={"dbname": DBNAME_CONSTRAINTS}) + self._map.create_regs() + return resource_cls + + def match(self, path): + params = self._map.match(path) + if params is None: + return None, None + resource_cls = params.pop('resource_cls') + return resource_cls, params + +url_to_resource = URLToResource() + + +@url_to_resource.register +class GlobalResource(object): + """Global (root) resource.""" + + url_pattern = "/" + + def __init__(self, state, responder): + self.state = state + self.responder = responder + + @http_method() + def get(self): + info = self.state.global_info() + info['version'] = _u1db_version + self.responder.send_response_json(**info) + + +@url_to_resource.register +class DatabaseResource(object): + """Database resource.""" + + url_pattern = "/{dbname}" + + def __init__(self, dbname, state, responder): + self.dbname = dbname + self.state = state + self.responder = responder + + @http_method() + def get(self): + self.state.check_database(self.dbname) + self.responder.send_response_json(200) + + @http_method(content_as_args=True) + def put(self): + self.state.ensure_database(self.dbname) + self.responder.send_response_json(200, ok=True) + + @http_method() + def delete(self): + self.state.delete_database(self.dbname) + self.responder.send_response_json(200, ok=True) + + +@url_to_resource.register +class DocsResource(object): + """Documents resource.""" + + url_pattern = "/{dbname}/docs" + + def __init__(self, dbname, state, responder): + self.responder = responder + self.db = state.open_database(dbname) + + @http_method(doc_ids=parse_list, check_for_conflicts=parse_bool, + include_deleted=parse_bool) + def get(self, doc_ids=None, check_for_conflicts=True, + include_deleted=False): + if doc_ids is None: + raise errors.MissingDocIds + docs = self.db.get_docs(doc_ids, include_deleted=include_deleted) + self.responder.content_type = 'application/json' + self.responder.start_response(200) + self.responder.start_stream(), + for doc in docs: + entry = dict( + doc_id=doc.doc_id, doc_rev=doc.rev, content=doc.get_json(), + has_conflicts=doc.has_conflicts) + self.responder.stream_entry(entry) + self.responder.end_stream() + self.responder.finish_response() + + +@url_to_resource.register +class AllDocsResource(object): + """All Documents resource.""" + + url_pattern = "/{dbname}/all-docs" + + def __init__(self, dbname, state, responder): + self.responder = responder + self.db = state.open_database(dbname) + + @http_method(include_deleted=parse_bool) + def get(self, include_deleted=False): + gen, docs = self.db.get_all_docs(include_deleted=include_deleted) + self.responder.content_type = 'application/json' + # returning a x-u1db-generation header is optional + # HTTPDatabase will fallback to return -1 if it's missing + self.responder.start_response(200, + headers={'x-u1db-generation': str(gen)}) + self.responder.start_stream(), + for doc in docs: + entry = dict( + doc_id=doc.doc_id, doc_rev=doc.rev, content=doc.get_json(), + has_conflicts=doc.has_conflicts) + self.responder.stream_entry(entry) + self.responder.end_stream() + self.responder.finish_response() + + +@url_to_resource.register +class DocResource(object): + """Document resource.""" + + url_pattern = "/{dbname}/doc/{id:.*}" + + def __init__(self, dbname, id, state, responder): + self.id = id + self.responder = responder + self.db = state.open_database(dbname) + + @http_method(old_rev=str) + def put(self, content, old_rev=None): + doc = Document(self.id, old_rev, content) + doc_rev = self.db.put_doc(doc) + if old_rev is None: + status = 201 # created + else: + status = 200 + self.responder.send_response_json(status, rev=doc_rev) + + @http_method(old_rev=str) + def delete(self, old_rev=None): + doc = Document(self.id, old_rev, None) + self.db.delete_doc(doc) + self.responder.send_response_json(200, rev=doc.rev) + + @http_method(include_deleted=parse_bool) + def get(self, include_deleted=False): + doc = self.db.get_doc(self.id, include_deleted=include_deleted) + if doc is None: + wire_descr = errors.DocumentDoesNotExist.wire_description + self.responder.send_response_json( + http_errors.wire_description_to_status[wire_descr], + error=wire_descr, + headers={ + 'x-u1db-rev': '', + 'x-u1db-has-conflicts': 'false' + }) + return + headers = { + 'x-u1db-rev': doc.rev, + 'x-u1db-has-conflicts': json.dumps(doc.has_conflicts) + } + if doc.is_tombstone(): + self.responder.send_response_json( + http_errors.wire_description_to_status[ + errors.DOCUMENT_DELETED], + error=errors.DOCUMENT_DELETED, + headers=headers) + else: + self.responder.send_response_content( + doc.get_json(), headers=headers) + + +@url_to_resource.register +class SyncResource(object): + """Sync endpoint resource.""" + + # maximum allowed request body size + max_request_size = 15 * 1024 * 1024 # 15Mb + # maximum allowed entry/line size in request body + max_entry_size = 10 * 1024 * 1024 # 10Mb + + url_pattern = "/{dbname}/sync-from/{source_replica_uid}" + + # pluggable + sync_exchange_class = sync.SyncExchange + + def __init__(self, dbname, source_replica_uid, state, responder): + self.source_replica_uid = source_replica_uid + self.responder = responder + self.state = state + self.dbname = dbname + self.replica_uid = None + + def get_target(self): + return self.state.open_database(self.dbname).get_sync_target() + + @http_method() + def get(self): + result = self.get_target().get_sync_info(self.source_replica_uid) + self.responder.send_response_json( + target_replica_uid=result[0], target_replica_generation=result[1], + target_replica_transaction_id=result[2], + source_replica_uid=self.source_replica_uid, + source_replica_generation=result[3], + source_transaction_id=result[4]) + + @http_method(generation=int, + content_as_args=True, no_query=True) + def put(self, generation, transaction_id): + self.get_target().record_sync_info(self.source_replica_uid, + generation, + transaction_id) + self.responder.send_response_json(ok=True) + + # Implements the same logic as LocalSyncTarget.sync_exchange + + @http_method(last_known_generation=int, last_known_trans_id=none_or_str, + content_as_args=True) + def post_args(self, last_known_generation, last_known_trans_id=None, + ensure=False): + if ensure: + db, self.replica_uid = self.state.ensure_database(self.dbname) + else: + db = self.state.open_database(self.dbname) + db.validate_gen_and_trans_id( + last_known_generation, last_known_trans_id) + self.sync_exch = self.sync_exchange_class( + db, self.source_replica_uid, last_known_generation) + + @http_method(content_as_args=True) + def post_stream_entry(self, id, rev, content, gen, trans_id): + doc = Document(id, rev, content) + self.sync_exch.insert_doc_from_source(doc, gen, trans_id) + + def post_end(self): + + def send_doc(doc, gen, trans_id): + entry = dict(id=doc.doc_id, rev=doc.rev, content=doc.get_json(), + gen=gen, trans_id=trans_id) + self.responder.stream_entry(entry) + + new_gen = self.sync_exch.find_changes_to_return() + self.responder.content_type = 'application/x-u1db-sync-stream' + self.responder.start_response(200) + self.responder.start_stream(), + header = {"new_generation": new_gen, + "new_transaction_id": self.sync_exch.new_trans_id} + if self.replica_uid is not None: + header['replica_uid'] = self.replica_uid + self.responder.stream_entry(header) + self.sync_exch.return_docs(send_doc) + self.responder.end_stream() + self.responder.finish_response() + + +class HTTPResponder(object): + """Encode responses from the server back to the client.""" + + # a multi document response will put args and documents + # each on one line of the response body + + def __init__(self, start_response): + self._started = False + self._stream_state = -1 + self._no_initial_obj = True + self.sent_response = False + self._start_response = start_response + self._write = None + self.content_type = 'application/json' + self.content = [] + + def start_response(self, status, obj_dic=None, headers={}): + """start sending response with optional first json object.""" + if self._started: + return + self._started = True + status_text = httplib.responses[status] + self._write = self._start_response('%d %s' % (status, status_text), + [('content-type', self.content_type), + ('cache-control', 'no-cache')] + + headers.items()) + # xxx version in headers + if obj_dic is not None: + self._no_initial_obj = False + self._write(json.dumps(obj_dic) + "\r\n") + + def finish_response(self): + """finish sending response.""" + self.sent_response = True + + def send_response_json(self, status=200, headers={}, **kwargs): + """send and finish response with json object body from keyword args.""" + content = json.dumps(kwargs) + "\r\n" + self.send_response_content(content, headers=headers, status=status) + + def send_response_content(self, content, status=200, headers={}): + """send and finish response with content""" + headers['content-length'] = str(len(content)) + self.start_response(status, headers=headers) + if self._stream_state == 1: + self.content = [',\r\n', content] + else: + self.content = [content] + self.finish_response() + + def start_stream(self): + "start stream (array) as part of the response." + assert self._started and self._no_initial_obj + self._stream_state = 0 + self._write("[") + + def stream_entry(self, entry): + "send stream entry as part of the response." + assert self._stream_state != -1 + if self._stream_state == 0: + self._stream_state = 1 + self._write('\r\n') + else: + self._write(',\r\n') + self._write(json.dumps(entry)) + + def end_stream(self): + "end stream (array)." + assert self._stream_state != -1 + self._write("\r\n]\r\n") + + +class HTTPInvocationByMethodWithBody(object): + """Invoke methods on a resource.""" + + def __init__(self, resource, environ, parameters): + self.resource = resource + self.environ = environ + self.max_request_size = getattr( + resource, 'max_request_size', parameters.max_request_size) + self.max_entry_size = getattr( + resource, 'max_entry_size', parameters.max_entry_size) + + def _lookup(self, method): + try: + return getattr(self.resource, method) + except AttributeError: + raise BadRequest() + + def __call__(self): + args = urlparse.parse_qsl(self.environ['QUERY_STRING'], + strict_parsing=False) + try: + args = dict( + (k.decode('utf-8'), v.decode('utf-8')) for k, v in args) + except ValueError: + raise BadRequest() + method = self.environ['REQUEST_METHOD'].lower() + if method in ('get', 'delete'): + meth = self._lookup(method) + return meth(args, None) + else: + # we expect content-length > 0, reconsider if we move + # to support chunked enconding + try: + content_length = int(self.environ['CONTENT_LENGTH']) + except (ValueError, KeyError): + raise BadRequest + if content_length <= 0: + raise BadRequest + if content_length > self.max_request_size: + raise BadRequest + reader = _FencedReader(self.environ['wsgi.input'], content_length, + self.max_entry_size) + content_type = self.environ.get('CONTENT_TYPE', '') + content_type = content_type.split(';', 1)[0].strip() + if content_type == 'application/json': + meth = self._lookup(method) + body = reader.read_chunk(sys.maxint) + return meth(args, body) + elif content_type == 'application/x-u1db-sync-stream': + meth_args = self._lookup('%s_args' % method) + meth_entry = self._lookup('%s_stream_entry' % method) + meth_end = self._lookup('%s_end' % method) + body_getline = reader.getline + if body_getline().strip() != '[': + raise BadRequest() + line = body_getline() + line, comma = utils.check_and_strip_comma(line.strip()) + meth_args(args, line) + while True: + line = body_getline() + entry = line.strip() + if entry == ']': + break + if not entry or not comma: # empty or no prec comma + raise BadRequest + entry, comma = utils.check_and_strip_comma(entry) + meth_entry({}, entry) + if comma or body_getline(): # extra comma or data + raise BadRequest + return meth_end() + else: + raise BadRequest() + + +class HTTPApp(object): + + # maximum allowed request body size + max_request_size = 15 * 1024 * 1024 # 15Mb + # maximum allowed entry/line size in request body + max_entry_size = 10 * 1024 * 1024 # 10Mb + + def __init__(self, state): + self.state = state + + def _lookup_resource(self, environ, responder): + resource_cls, params = url_to_resource.match(environ['PATH_INFO']) + if resource_cls is None: + raise BadRequest # 404 instead? + resource = resource_cls( + state=self.state, responder=responder, **params) + return resource + + def __call__(self, environ, start_response): + responder = HTTPResponder(start_response) + self.request_begin(environ) + try: + resource = self._lookup_resource(environ, responder) + HTTPInvocationByMethodWithBody(resource, environ, self)() + except errors.U1DBError, e: + self.request_u1db_error(environ, e) + status = http_errors.wire_description_to_status.get( + e.wire_description, 500) + responder.send_response_json(status, error=e.wire_description) + except BadRequest: + self.request_bad_request(environ) + responder.send_response_json(400, error="bad request") + except KeyboardInterrupt: + raise + except: + self.request_failed(environ) + raise + else: + self.request_done(environ) + return responder.content + + # hooks for tracing requests + + def request_begin(self, environ): + """Hook called at the beginning of processing a request.""" + pass + + def request_done(self, environ): + """Hook called when done processing a request.""" + pass + + def request_u1db_error(self, environ, exc): + """Hook called when processing a request resulted in a U1DBError. + + U1DBError passed as exc. + """ + pass + + def request_bad_request(self, environ): + """Hook called when processing a bad request. + + No actual processing was done. + """ + pass + + def request_failed(self, environ): + """Hook called when processing a request failed unexpectedly. + + Invoked from an except block, so there's interpreter exception + information available. + """ + pass diff --git a/common/src/leap/soledad/common/l2db/remote/http_client.py b/common/src/leap/soledad/common/l2db/remote/http_client.py new file mode 100644 index 00000000..2044d756 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/remote/http_client.py @@ -0,0 +1,219 @@ +# Copyright 2011-2012 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +"""Base class to make requests to a remote HTTP server.""" + +import httplib +from oauth import oauth +try: + import simplejson as json +except ImportError: + import json # noqa +import socket +import ssl +import sys +import urlparse +import urllib + +from time import sleep +from u1db import ( + errors, + ) +from u1db.remote import ( + http_errors, + ) + +from u1db.remote.ssl_match_hostname import ( # noqa + CertificateError, + match_hostname, + ) + +# Ubuntu/debian +# XXX other... +CA_CERTS = "/etc/ssl/certs/ca-certificates.crt" + + +def _encode_query_parameter(value): + """Encode query parameter.""" + if isinstance(value, bool): + if value: + value = 'true' + else: + value = 'false' + return unicode(value).encode('utf-8') + + +class _VerifiedHTTPSConnection(httplib.HTTPSConnection): + """HTTPSConnection verifying server side certificates.""" + # derived from httplib.py + + def connect(self): + "Connect to a host on a given (SSL) port." + + sock = socket.create_connection((self.host, self.port), + self.timeout, self.source_address) + if self._tunnel_host: + self.sock = sock + self._tunnel() + if sys.platform.startswith('linux'): + cert_opts = { + 'cert_reqs': ssl.CERT_REQUIRED, + 'ca_certs': CA_CERTS + } + else: + # XXX no cert verification implemented elsewhere for now + cert_opts = {} + self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file, + ssl_version=ssl.PROTOCOL_SSLv3, + **cert_opts + ) + if cert_opts: + match_hostname(self.sock.getpeercert(), self.host) + + +class HTTPClientBase(object): + """Base class to make requests to a remote HTTP server.""" + + # by default use HMAC-SHA1 OAuth signature method to not disclose + # tokens + # NB: given that the content bodies are not covered by the + # signatures though, to achieve security (against man-in-the-middle + # attacks for example) one would need HTTPS + oauth_signature_method = oauth.OAuthSignatureMethod_HMAC_SHA1() + + # Will use these delays to retry on 503 befor finally giving up. The final + # 0 is there to not wait after the final try fails. + _delays = (1, 1, 2, 4, 0) + + def __init__(self, url, creds=None): + self._url = urlparse.urlsplit(url) + self._conn = None + self._creds = {} + if creds is not None: + if len(creds) != 1: + raise errors.UnknownAuthMethod() + auth_meth, credentials = creds.items()[0] + try: + set_creds = getattr(self, 'set_%s_credentials' % auth_meth) + except AttributeError: + raise errors.UnknownAuthMethod(auth_meth) + set_creds(**credentials) + + def set_oauth_credentials(self, consumer_key, consumer_secret, + token_key, token_secret): + self._creds = {'oauth': ( + oauth.OAuthConsumer(consumer_key, consumer_secret), + oauth.OAuthToken(token_key, token_secret))} + + def _ensure_connection(self): + if self._conn is not None: + return + if self._url.scheme == 'https': + connClass = _VerifiedHTTPSConnection + else: + connClass = httplib.HTTPConnection + self._conn = connClass(self._url.hostname, self._url.port) + + def close(self): + if self._conn: + self._conn.close() + self._conn = None + + # xxx retry mechanism? + + def _error(self, respdic): + descr = respdic.get("error") + exc_cls = errors.wire_description_to_exc.get(descr) + if exc_cls is not None: + message = respdic.get("message") + raise exc_cls(message) + + def _response(self): + resp = self._conn.getresponse() + body = resp.read() + headers = dict(resp.getheaders()) + if resp.status in (200, 201): + return body, headers + elif resp.status in http_errors.ERROR_STATUSES: + try: + respdic = json.loads(body) + except ValueError: + pass + else: + print "ERROR--->", respdic + self._error(respdic) + # special case + if resp.status == 503: + raise errors.Unavailable(body, headers) + raise errors.HTTPError(resp.status, body, headers) + + def _sign_request(self, method, url_query, params): + if 'oauth' in self._creds: + consumer, token = self._creds['oauth'] + full_url = "%s://%s%s" % (self._url.scheme, self._url.netloc, + url_query) + oauth_req = oauth.OAuthRequest.from_consumer_and_token( + consumer, token, + http_method=method, + parameters=params, + http_url=full_url + ) + oauth_req.sign_request( + self.oauth_signature_method, consumer, token) + # Authorization: OAuth ... + return oauth_req.to_header().items() + else: + return [] + + def _request(self, method, url_parts, params=None, body=None, + content_type=None): + self._ensure_connection() + unquoted_url = url_query = self._url.path + if url_parts: + if not url_query.endswith('/'): + url_query += '/' + unquoted_url = url_query + url_query += '/'.join(urllib.quote(part, safe='') + for part in url_parts) + # oauth performs its own quoting + unquoted_url += '/'.join(url_parts) + encoded_params = {} + if params: + for key, value in params.items(): + key = unicode(key).encode('utf-8') + encoded_params[key] = _encode_query_parameter(value) + url_query += ('?' + urllib.urlencode(encoded_params)) + if body is not None and not isinstance(body, basestring): + body = json.dumps(body) + content_type = 'application/json' + headers = {} + if content_type: + headers['content-type'] = content_type + headers.update( + self._sign_request(method, unquoted_url, encoded_params)) + for delay in self._delays: + try: + self._conn.request(method, url_query, body, headers) + return self._response() + except errors.Unavailable, e: + sleep(delay) + raise e + + def _request_json(self, method, url_parts, params=None, body=None, + content_type=None): + res, headers = self._request(method, url_parts, params, body, + content_type) + return json.loads(res), headers diff --git a/common/src/leap/soledad/common/l2db/remote/http_database.py b/common/src/leap/soledad/common/l2db/remote/http_database.py new file mode 100644 index 00000000..400e4020 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/remote/http_database.py @@ -0,0 +1,163 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +"""HTTPDatabase to access a remote db over the HTTP API.""" + +try: + import simplejson as json +except ImportError: + import json # noqa +import uuid + +from u1db import ( + Database, + Document, + errors, + ) +from u1db.remote import ( + http_client, + http_errors, + http_target, + ) + + +DOCUMENT_DELETED_STATUS = http_errors.wire_description_to_status[ + errors.DOCUMENT_DELETED] + + +class HTTPDatabase(http_client.HTTPClientBase, Database): + """Implement the Database API to a remote HTTP server.""" + + def __init__(self, url, document_factory=None, creds=None): + super(HTTPDatabase, self).__init__(url, creds=creds) + self._factory = document_factory or Document + + def set_document_factory(self, factory): + self._factory = factory + + @staticmethod + def open_database(url, create): + db = HTTPDatabase(url) + db.open(create) + return db + + @staticmethod + def delete_database(url): + db = HTTPDatabase(url) + db._delete() + db.close() + + def open(self, create): + if create: + self._ensure() + else: + self._check() + + def _check(self): + return self._request_json('GET', [])[0] + + def _ensure(self): + self._request_json('PUT', [], {}, {}) + + def _delete(self): + self._request_json('DELETE', [], {}, {}) + + def put_doc(self, doc): + if doc.doc_id is None: + raise errors.InvalidDocId() + params = {} + if doc.rev is not None: + params['old_rev'] = doc.rev + res, headers = self._request_json('PUT', ['doc', doc.doc_id], params, + doc.get_json(), 'application/json') + doc.rev = res['rev'] + return res['rev'] + + def get_doc(self, doc_id, include_deleted=False): + try: + res, headers = self._request( + 'GET', ['doc', doc_id], {"include_deleted": include_deleted}) + except errors.DocumentDoesNotExist: + return None + except errors.HTTPError, e: + if (e.status == DOCUMENT_DELETED_STATUS and + 'x-u1db-rev' in e.headers): + res = None + headers = e.headers + else: + raise + doc_rev = headers['x-u1db-rev'] + has_conflicts = json.loads(headers['x-u1db-has-conflicts']) + doc = self._factory(doc_id, doc_rev, res) + doc.has_conflicts = has_conflicts + return doc + + def _build_docs(self, res): + for doc_dict in json.loads(res): + doc = self._factory( + doc_dict['doc_id'], doc_dict['doc_rev'], doc_dict['content']) + doc.has_conflicts = doc_dict['has_conflicts'] + yield doc + + def get_docs(self, doc_ids, check_for_conflicts=True, + include_deleted=False): + if not doc_ids: + return [] + doc_ids = ','.join(doc_ids) + res, headers = self._request( + 'GET', ['docs'], { + "doc_ids": doc_ids, "include_deleted": include_deleted, + "check_for_conflicts": check_for_conflicts}) + return self._build_docs(res) + + def get_all_docs(self, include_deleted=False): + res, headers = self._request( + 'GET', ['all-docs'], {"include_deleted": include_deleted}) + gen = -1 + if 'x-u1db-generation' in headers: + gen = int(headers['x-u1db-generation']) + return gen, list(self._build_docs(res)) + + def _allocate_doc_id(self): + return 'D-%s' % (uuid.uuid4().hex,) + + def create_doc(self, content, doc_id=None): + if not isinstance(content, dict): + raise errors.InvalidContent + json_string = json.dumps(content) + return self.create_doc_from_json(json_string, doc_id) + + def create_doc_from_json(self, content, doc_id=None): + if doc_id is None: + doc_id = self._allocate_doc_id() + res, headers = self._request_json('PUT', ['doc', doc_id], {}, + content, 'application/json') + new_doc = self._factory(doc_id, res['rev'], content) + return new_doc + + def delete_doc(self, doc): + if doc.doc_id is None: + raise errors.InvalidDocId() + params = {'old_rev': doc.rev} + res, headers = self._request_json('DELETE', + ['doc', doc.doc_id], params) + doc.make_tombstone() + doc.rev = res['rev'] + + def get_sync_target(self): + st = http_target.HTTPSyncTarget(self._url.geturl()) + st._creds = self._creds + return st diff --git a/common/src/leap/soledad/common/l2db/remote/http_errors.py b/common/src/leap/soledad/common/l2db/remote/http_errors.py new file mode 100644 index 00000000..f6bd01d3 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/remote/http_errors.py @@ -0,0 +1,47 @@ +# Copyright 2011-2012 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +"""Information about the encoding of errors over HTTP.""" + +from u1db import ( + errors, + ) + + +# error wire descriptions mapping to HTTP status codes +wire_description_to_status = dict([ + (errors.InvalidDocId.wire_description, 400), + (errors.MissingDocIds.wire_description, 400), + (errors.Unauthorized.wire_description, 401), + (errors.DocumentTooBig.wire_description, 403), + (errors.UserQuotaExceeded.wire_description, 403), + (errors.SubscriptionNeeded.wire_description, 403), + (errors.DatabaseDoesNotExist.wire_description, 404), + (errors.DocumentDoesNotExist.wire_description, 404), + (errors.DocumentAlreadyDeleted.wire_description, 404), + (errors.RevisionConflict.wire_description, 409), + (errors.InvalidGeneration.wire_description, 409), + (errors.InvalidReplicaUID.wire_description, 409), + (errors.InvalidTransactionId.wire_description, 409), + (errors.Unavailable.wire_description, 503), +# without matching exception + (errors.DOCUMENT_DELETED, 404) +]) + + +ERROR_STATUSES = set(wire_description_to_status.values()) +# 400 included explicitly for tests +ERROR_STATUSES.add(400) diff --git a/common/src/leap/soledad/common/l2db/remote/http_target.py b/common/src/leap/soledad/common/l2db/remote/http_target.py new file mode 100644 index 00000000..1028963e --- /dev/null +++ b/common/src/leap/soledad/common/l2db/remote/http_target.py @@ -0,0 +1,135 @@ +# Copyright 2011-2012 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +"""SyncTarget API implementation to a remote HTTP server.""" + +try: + import simplejson as json +except ImportError: + import json # noqa + +from u1db import ( + Document, + SyncTarget, + ) +from u1db.errors import ( + BrokenSyncStream, + ) +from u1db.remote import ( + http_client, + utils, + ) + + +class HTTPSyncTarget(http_client.HTTPClientBase, SyncTarget): + """Implement the SyncTarget api to a remote HTTP server.""" + + @staticmethod + def connect(url): + return HTTPSyncTarget(url) + + def get_sync_info(self, source_replica_uid): + self._ensure_connection() + res, _ = self._request_json('GET', ['sync-from', source_replica_uid]) + return (res['target_replica_uid'], res['target_replica_generation'], + res['target_replica_transaction_id'], + res['source_replica_generation'], res['source_transaction_id']) + + def record_sync_info(self, source_replica_uid, source_replica_generation, + source_transaction_id): + self._ensure_connection() + if self._trace_hook: # for tests + self._trace_hook('record_sync_info') + self._request_json('PUT', ['sync-from', source_replica_uid], {}, + {'generation': source_replica_generation, + 'transaction_id': source_transaction_id}) + + def _parse_sync_stream(self, data, return_doc_cb, ensure_callback=None): + parts = data.splitlines() # one at a time + if not parts or parts[0] != '[': + raise BrokenSyncStream + data = parts[1:-1] + comma = False + if data: + line, comma = utils.check_and_strip_comma(data[0]) + res = json.loads(line) + if ensure_callback and 'replica_uid' in res: + ensure_callback(res['replica_uid']) + for entry in data[1:]: + if not comma: # missing in between comma + raise BrokenSyncStream + line, comma = utils.check_and_strip_comma(entry) + entry = json.loads(line) + doc = Document(entry['id'], entry['rev'], entry['content']) + return_doc_cb(doc, entry['gen'], entry['trans_id']) + if parts[-1] != ']': + try: + partdic = json.loads(parts[-1]) + except ValueError: + pass + else: + if isinstance(partdic, dict): + self._error(partdic) + raise BrokenSyncStream + if not data or comma: # no entries or bad extra comma + raise BrokenSyncStream + return res + + def sync_exchange(self, docs_by_generations, source_replica_uid, + last_known_generation, last_known_trans_id, + return_doc_cb, ensure_callback=None): + self._ensure_connection() + if self._trace_hook: # for tests + self._trace_hook('sync_exchange') + url = '%s/sync-from/%s' % (self._url.path, source_replica_uid) + self._conn.putrequest('POST', url) + self._conn.putheader('content-type', 'application/x-u1db-sync-stream') + for header_name, header_value in self._sign_request('POST', url, {}): + self._conn.putheader(header_name, header_value) + entries = ['['] + size = 1 + + def prepare(**dic): + entry = comma + '\r\n' + json.dumps(dic) + entries.append(entry) + return len(entry) + + comma = '' + size += prepare( + last_known_generation=last_known_generation, + last_known_trans_id=last_known_trans_id, + ensure=ensure_callback is not None) + comma = ',' + for doc, gen, trans_id in docs_by_generations: + size += prepare(id=doc.doc_id, rev=doc.rev, content=doc.get_json(), + gen=gen, trans_id=trans_id) + entries.append('\r\n]') + size += len(entries[-1]) + self._conn.putheader('content-length', str(size)) + self._conn.endheaders() + for entry in entries: + self._conn.send(entry) + entries = None + data, _ = self._response() + res = self._parse_sync_stream(data, return_doc_cb, ensure_callback) + data = None + return res['new_generation'], res['new_transaction_id'] + + # for tests + _trace_hook = None + + def _set_trace_hook_shallow(self, cb): + self._trace_hook = cb diff --git a/common/src/leap/soledad/common/l2db/remote/oauth_middleware.py b/common/src/leap/soledad/common/l2db/remote/oauth_middleware.py new file mode 100644 index 00000000..5772580a --- /dev/null +++ b/common/src/leap/soledad/common/l2db/remote/oauth_middleware.py @@ -0,0 +1,89 @@ +# Copyright 2012 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . +"""U1DB OAuth authorisation WSGI middleware.""" +import httplib +from oauth import oauth +try: + import simplejson as json +except ImportError: + import json # noqa +from urllib import quote +from wsgiref.util import shift_path_info + + +sign_meth_HMAC_SHA1 = oauth.OAuthSignatureMethod_HMAC_SHA1() +sign_meth_PLAINTEXT = oauth.OAuthSignatureMethod_PLAINTEXT() + + +class OAuthMiddleware(object): + """U1DB OAuth Authorisation WSGI middleware.""" + + # max seconds the request timestamp is allowed to be shifted + # from arrival time + timestamp_threshold = 300 + + def __init__(self, app, base_url, prefix='/~/'): + self.app = app + self.base_url = base_url + self.prefix = prefix + + def get_oauth_data_store(self): + """Provide a oauth.OAuthDataStore.""" + raise NotImplementedError(self.get_oauth_data_store) + + def _error(self, start_response, status, description, message=None): + start_response("%d %s" % (status, httplib.responses[status]), + [('content-type', 'application/json')]) + err = {"error": description} + if message: + err['message'] = message + return [json.dumps(err)] + + def __call__(self, environ, start_response): + if self.prefix and not environ['PATH_INFO'].startswith(self.prefix): + return self._error(start_response, 400, "bad request") + headers = {} + if 'HTTP_AUTHORIZATION' in environ: + headers['Authorization'] = environ['HTTP_AUTHORIZATION'] + oauth_req = oauth.OAuthRequest.from_request( + http_method=environ['REQUEST_METHOD'], + http_url=self.base_url + environ['PATH_INFO'], + headers=headers, + query_string=environ['QUERY_STRING'] + ) + if oauth_req is None: + return self._error(start_response, 401, "unauthorized", + "Missing OAuth.") + try: + self.verify(environ, oauth_req) + except oauth.OAuthError, e: + return self._error(start_response, 401, "unauthorized", + e.message) + shift_path_info(environ) + return self.app(environ, start_response) + + def verify(self, environ, oauth_req): + """Verify OAuth request, put user_id in the environ.""" + oauth_server = oauth.OAuthServer(self.get_oauth_data_store()) + oauth_server.timestamp_threshold = self.timestamp_threshold + oauth_server.add_signature_method(sign_meth_HMAC_SHA1) + oauth_server.add_signature_method(sign_meth_PLAINTEXT) + consumer, token, parameters = oauth_server.verify_request(oauth_req) + # filter out oauth bits + environ['QUERY_STRING'] = '&'.join("%s=%s" % (quote(k, safe=''), + quote(v, safe='')) + for k, v in parameters.iteritems()) + return consumer, token diff --git a/common/src/leap/soledad/common/l2db/remote/server_state.py b/common/src/leap/soledad/common/l2db/remote/server_state.py new file mode 100644 index 00000000..6c1104c6 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/remote/server_state.py @@ -0,0 +1,71 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +"""State for servers exposing a set of U1DB databases.""" +import os +import errno + +class ServerState(object): + """Passed to a Request when it is instantiated. + + This is used to track server-side state, such as working-directory, open + databases, etc. + """ + + def __init__(self): + self._workingdir = None + + def set_workingdir(self, path): + self._workingdir = path + + def global_info(self): + """Return global information about the server.""" + return {} + + def _relpath(self, relpath): + # Note: We don't want to allow absolute paths here, because we + # don't want to expose the filesystem. We should also check that + # relpath doesn't have '..' in it, etc. + return self._workingdir + '/' + relpath + + def open_database(self, path): + """Open a database at the given location.""" + from u1db.backends import sqlite_backend + full_path = self._relpath(path) + return sqlite_backend.SQLiteDatabase.open_database(full_path, + create=False) + + def check_database(self, path): + """Check if the database at the given location exists. + + Simply returns if it does or raises DatabaseDoesNotExist. + """ + db = self.open_database(path) + db.close() + + def ensure_database(self, path): + """Ensure database at the given location.""" + from u1db.backends import sqlite_backend + full_path = self._relpath(path) + db = sqlite_backend.SQLiteDatabase.open_database(full_path, + create=True) + return db, db._replica_uid + + def delete_database(self, path): + """Delete database at the given location.""" + from u1db.backends import sqlite_backend + full_path = self._relpath(path) + sqlite_backend.SQLiteDatabase.delete_database(full_path) diff --git a/common/src/leap/soledad/common/l2db/remote/ssl_match_hostname.py b/common/src/leap/soledad/common/l2db/remote/ssl_match_hostname.py new file mode 100644 index 00000000..fbabc177 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/remote/ssl_match_hostname.py @@ -0,0 +1,64 @@ +"""The match_hostname() function from Python 3.2, essential when using SSL.""" +# XXX put it here until it's packaged + +import re + +__version__ = '3.2a3' + + +class CertificateError(ValueError): + pass + + +def _dnsname_to_pat(dn): + pats = [] + for frag in dn.split(r'.'): + if frag == '*': + # When '*' is a fragment by itself, it matches a non-empty dotless + # fragment. + pats.append('[^.]+') + else: + # Otherwise, '*' matches any dotless fragment. + frag = re.escape(frag) + pats.append(frag.replace(r'\*', '[^.]*')) + return re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) + + +def match_hostname(cert, hostname): + """Verify that *cert* (in decoded format as returned by + SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 rules + are mostly followed, but IP addresses are not accepted for *hostname*. + + CertificateError is raised on failure. On success, the function + returns nothing. + """ + if not cert: + raise ValueError("empty or no certificate") + dnsnames = [] + san = cert.get('subjectAltName', ()) + for key, value in san: + if key == 'DNS': + if _dnsname_to_pat(value).match(hostname): + return + dnsnames.append(value) + if not san: + # The subject is only checked when subjectAltName is empty + for sub in cert.get('subject', ()): + for key, value in sub: + # XXX according to RFC 2818, the most specific Common Name + # must be used. + if key == 'commonName': + if _dnsname_to_pat(value).match(hostname): + return + dnsnames.append(value) + if len(dnsnames) > 1: + raise CertificateError("hostname %r " + "doesn't match either of %s" + % (hostname, ', '.join(map(repr, dnsnames)))) + elif len(dnsnames) == 1: + raise CertificateError("hostname %r " + "doesn't match %r" + % (hostname, dnsnames[0])) + else: + raise CertificateError("no appropriate commonName or " + "subjectAltName fields were found") diff --git a/common/src/leap/soledad/common/l2db/remote/utils.py b/common/src/leap/soledad/common/l2db/remote/utils.py new file mode 100644 index 00000000..14cedea9 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/remote/utils.py @@ -0,0 +1,23 @@ +# Copyright 2012 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +"""Utilities for details of the procotol.""" + + +def check_and_strip_comma(line): + if line and line[-1] == ',': + return line[:-1], True + return line, False diff --git a/common/src/leap/soledad/common/l2db/sync.py b/common/src/leap/soledad/common/l2db/sync.py new file mode 100644 index 00000000..d9e455d8 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/sync.py @@ -0,0 +1,308 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +"""The synchronization utilities for U1DB.""" +from itertools import izip + +import u1db +from u1db import errors + + +class Synchronizer(object): + """Collect the state around synchronizing 2 U1DB replicas. + + Synchronization is bi-directional, in that new items in the source are sent + to the target, and new items in the target are returned to the source. + However, it still recognizes that one side is initiating the request. Also, + at the moment, conflicts are only created in the source. + """ + + def __init__(self, source, sync_target): + """Create a new Synchronization object. + + :param source: A Database + :param sync_target: A SyncTarget + """ + self.source = source + self.sync_target = sync_target + self.target_replica_uid = None + self.num_inserted = 0 + + def _insert_doc_from_target(self, doc, replica_gen, trans_id): + """Try to insert synced document from target. + + Implements TAKE OTHER semantics: any document from the target + that is in conflict will be taken as the new official value, + while the current conflicting value will be stored alongside + as a conflict. In the process indexes will be updated etc. + + :return: None + """ + # Increases self.num_inserted depending whether the document + # was effectively inserted. + state, _ = self.source._put_doc_if_newer(doc, save_conflict=True, + replica_uid=self.target_replica_uid, replica_gen=replica_gen, + replica_trans_id=trans_id) + if state == 'inserted': + self.num_inserted += 1 + elif state == 'converged': + # magical convergence + pass + elif state == 'superseded': + # we have something newer, will be taken care of at the next sync + pass + else: + assert state == 'conflicted' + # The doc was saved as a conflict, so the database was updated + self.num_inserted += 1 + + def _record_sync_info_with_the_target(self, start_generation): + """Record our new after sync generation with the target if gapless. + + Any documents received from the target will cause the local + database to increment its generation. We do not want to send + them back to the target in a future sync. However, there could + also be concurrent updates from another process doing eg + 'put_doc' while the sync was running. And we do want to + synchronize those documents. We can tell if there was a + concurrent update by comparing our new generation number + versus the generation we started, and how many documents we + inserted from the target. If it matches exactly, then we can + record with the target that they are fully up to date with our + new generation. + """ + cur_gen, trans_id = self.source._get_generation_info() + if (cur_gen == start_generation + self.num_inserted + and self.num_inserted > 0): + self.sync_target.record_sync_info( + self.source._replica_uid, cur_gen, trans_id) + + def sync(self, callback=None, autocreate=False): + """Synchronize documents between source and target.""" + sync_target = self.sync_target + # get target identifier, its current generation, + # and its last-seen database generation for this source + try: + (self.target_replica_uid, target_gen, target_trans_id, + target_my_gen, target_my_trans_id) = sync_target.get_sync_info( + self.source._replica_uid) + except errors.DatabaseDoesNotExist: + if not autocreate: + raise + # will try to ask sync_exchange() to create the db + self.target_replica_uid = None + target_gen, target_trans_id = 0, '' + target_my_gen, target_my_trans_id = 0, '' + + def ensure_callback(replica_uid): + self.target_replica_uid = replica_uid + + else: + ensure_callback = None + if self.target_replica_uid == self.source._replica_uid: + raise errors.InvalidReplicaUID + # validate the generation and transaction id the target knows about us + self.source.validate_gen_and_trans_id( + target_my_gen, target_my_trans_id) + # what's changed since that generation and this current gen + my_gen, _, changes = self.source.whats_changed(target_my_gen) + + # this source last-seen database generation for the target + if self.target_replica_uid is None: + target_last_known_gen, target_last_known_trans_id = 0, '' + else: + target_last_known_gen, target_last_known_trans_id = \ + self.source._get_replica_gen_and_trans_id(self.target_replica_uid) + if not changes and target_last_known_gen == target_gen: + if target_trans_id != target_last_known_trans_id: + raise errors.InvalidTransactionId + return my_gen + changed_doc_ids = [doc_id for doc_id, _, _ in changes] + # prepare to send all the changed docs + docs_to_send = self.source.get_docs(changed_doc_ids, + check_for_conflicts=False, include_deleted=True) + # TODO: there must be a way to not iterate twice + docs_by_generation = zip( + docs_to_send, (gen for _, gen, _ in changes), + (trans for _, _, trans in changes)) + + # exchange documents and try to insert the returned ones with + # the target, return target synced-up-to gen + new_gen, new_trans_id = sync_target.sync_exchange( + docs_by_generation, self.source._replica_uid, + target_last_known_gen, target_last_known_trans_id, + self._insert_doc_from_target, ensure_callback=ensure_callback) + # record target synced-up-to generation including applying what we sent + self.source._set_replica_gen_and_trans_id( + self.target_replica_uid, new_gen, new_trans_id) + + # if gapless record current reached generation with target + self._record_sync_info_with_the_target(my_gen) + + return my_gen + + +class SyncExchange(object): + """Steps and state for carrying through a sync exchange on a target.""" + + def __init__(self, db, source_replica_uid, last_known_generation): + self._db = db + self.source_replica_uid = source_replica_uid + self.source_last_known_generation = last_known_generation + self.seen_ids = {} # incoming ids not superseded + self.changes_to_return = None + self.new_gen = None + self.new_trans_id = None + # for tests + self._incoming_trace = [] + self._trace_hook = None + self._db._last_exchange_log = { + 'receive': {'docs': self._incoming_trace}, + 'return': None + } + + def _set_trace_hook(self, cb): + self._trace_hook = cb + + def _trace(self, state): + if not self._trace_hook: + return + self._trace_hook(state) + + def insert_doc_from_source(self, doc, source_gen, trans_id): + """Try to insert synced document from source. + + Conflicting documents are not inserted but will be sent over + to the sync source. + + It keeps track of progress by storing the document source + generation as well. + + The 1st step of a sync exchange is to call this repeatedly to + try insert all incoming documents from the source. + + :param doc: A Document object. + :param source_gen: The source generation of doc. + :return: None + """ + state, at_gen = self._db._put_doc_if_newer(doc, save_conflict=False, + replica_uid=self.source_replica_uid, replica_gen=source_gen, + replica_trans_id=trans_id) + if state == 'inserted': + self.seen_ids[doc.doc_id] = at_gen + elif state == 'converged': + # magical convergence + self.seen_ids[doc.doc_id] = at_gen + elif state == 'superseded': + # we have something newer that we will return + pass + else: + # conflict that we will returne + assert state == 'conflicted' + # for tests + self._incoming_trace.append((doc.doc_id, doc.rev)) + self._db._last_exchange_log['receive'].update({ + 'source_uid': self.source_replica_uid, + 'source_gen': source_gen + }) + + def find_changes_to_return(self): + """Find changes to return. + + Find changes since last_known_generation in db generation + order using whats_changed. It excludes documents ids that have + already been considered (superseded by the sender, etc). + + :return: new_generation - the generation of this database + which the caller can consider themselves to be synchronized after + processing the returned documents. + """ + self._db._last_exchange_log['receive'].update({ # for tests + 'last_known_gen': self.source_last_known_generation + }) + self._trace('before whats_changed') + gen, trans_id, changes = self._db.whats_changed( + self.source_last_known_generation) + self._trace('after whats_changed') + self.new_gen = gen + self.new_trans_id = trans_id + seen_ids = self.seen_ids + # changed docs that weren't superseded by or converged with + self.changes_to_return = [ + (doc_id, gen, trans_id) for (doc_id, gen, trans_id) in changes + # there was a subsequent update + if doc_id not in seen_ids or seen_ids.get(doc_id) < gen] + return self.new_gen + + def return_docs(self, return_doc_cb): + """Return the changed documents and their last change generation + repeatedly invoking the callback return_doc_cb. + + The final step of a sync exchange. + + :param: return_doc_cb(doc, gen, trans_id): is a callback + used to return the documents with their last change generation + to the target replica. + :return: None + """ + changes_to_return = self.changes_to_return + # return docs, including conflicts + changed_doc_ids = [doc_id for doc_id, _, _ in changes_to_return] + self._trace('before get_docs') + docs = self._db.get_docs( + changed_doc_ids, check_for_conflicts=False, include_deleted=True) + + docs_by_gen = izip( + docs, (gen for _, gen, _ in changes_to_return), + (trans_id for _, _, trans_id in changes_to_return)) + _outgoing_trace = [] # for tests + for doc, gen, trans_id in docs_by_gen: + return_doc_cb(doc, gen, trans_id) + _outgoing_trace.append((doc.doc_id, doc.rev)) + # for tests + self._db._last_exchange_log['return'] = { + 'docs': _outgoing_trace, + 'last_gen': self.new_gen + } + + +class LocalSyncTarget(u1db.SyncTarget): + """Common sync target implementation logic for all local sync targets.""" + + def __init__(self, db): + self._db = db + self._trace_hook = None + + def sync_exchange(self, docs_by_generations, source_replica_uid, + last_known_generation, last_known_trans_id, + return_doc_cb, ensure_callback=None): + self._db.validate_gen_and_trans_id( + last_known_generation, last_known_trans_id) + sync_exch = SyncExchange( + self._db, source_replica_uid, last_known_generation) + if self._trace_hook: + sync_exch._set_trace_hook(self._trace_hook) + # 1st step: try to insert incoming docs and record progress + for doc, doc_gen, trans_id in docs_by_generations: + sync_exch.insert_doc_from_source(doc, doc_gen, trans_id) + # 2nd step: find changed documents (including conflicts) to return + new_gen = sync_exch.find_changes_to_return() + # final step: return docs and record source replica sync point + sync_exch.return_docs(return_doc_cb) + return new_gen, sync_exch.new_trans_id + + def _set_trace_hook(self, cb): + self._trace_hook = cb diff --git a/common/src/leap/soledad/common/l2db/vectorclock.py b/common/src/leap/soledad/common/l2db/vectorclock.py new file mode 100644 index 00000000..42bceaa8 --- /dev/null +++ b/common/src/leap/soledad/common/l2db/vectorclock.py @@ -0,0 +1,89 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +"""VectorClockRev helper class.""" + + +class VectorClockRev(object): + """Track vector clocks for multiple replica ids. + + This allows simple comparison to determine if one VectorClockRev is + newer/older/in-conflict-with another VectorClockRev without having to + examine history. Every replica has a strictly increasing revision. When + creating a new revision, they include all revisions for all other replicas + which the new revision dominates, and increment their own revision to + something greater than the current value. + """ + + def __init__(self, value): + self._values = self._expand(value) + + def __repr__(self): + s = self.as_str() + return '%s(%s)' % (self.__class__.__name__, s) + + def as_str(self): + s = '|'.join(['%s:%d' % (m, r) for m, r + in sorted(self._values.items())]) + return s + + def _expand(self, value): + result = {} + if value is None: + return result + for replica_info in value.split('|'): + replica_uid, counter = replica_info.split(':') + counter = int(counter) + result[replica_uid] = counter + return result + + def is_newer(self, other): + """Is this VectorClockRev strictly newer than other. + """ + if not self._values: + return False + if not other._values: + return True + this_is_newer = False + other_expand = dict(other._values) + for key, value in self._values.iteritems(): + if key in other_expand: + other_value = other_expand.pop(key) + if other_value > value: + return False + elif other_value < value: + this_is_newer = True + else: + this_is_newer = True + if other_expand: + return False + return this_is_newer + + def increment(self, replica_uid): + """Increase the 'replica_uid' section of this vector clock. + + :return: A string representing the new vector clock value + """ + self._values[replica_uid] = self._values.get(replica_uid, 0) + 1 + + def maximize(self, other_vcr): + for replica_uid, counter in other_vcr._values.iteritems(): + if replica_uid not in self._values: + self._values[replica_uid] = counter + else: + this_counter = self._values[replica_uid] + if this_counter < counter: + self._values[replica_uid] = counter -- cgit v1.2.3 From b5aa97e9f88934dd73af84f212c95775f97769a9 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 28 Apr 2016 21:40:43 -0400 Subject: [refactor] make tests use l2db submodule From this moment on, we embed a fork of u1db called l2db. --- client/pkg/requirements.pip | 5 - client/src/leap/soledad/client/api.py | 5 +- client/src/leap/soledad/client/auth.py | 2 +- client/src/leap/soledad/client/http_target/api.py | 4 +- .../src/leap/soledad/client/http_target/fetch.py | 8 +- client/src/leap/soledad/client/http_target/send.py | 3 + .../src/leap/soledad/client/http_target/support.py | 5 +- client/src/leap/soledad/client/shared_db.py | 2 +- client/src/leap/soledad/client/sqlcipher.py | 11 +- client/src/leap/soledad/client/sync.py | 4 +- common/pkg/requirements.pip | 6 - common/src/leap/soledad/common/README.txt | 4 +- common/src/leap/soledad/common/backend.py | 15 +- common/src/leap/soledad/common/couch/__init__.py | 4 +- common/src/leap/soledad/common/couch/state.py | 10 +- common/src/leap/soledad/common/document.py | 4 +- common/src/leap/soledad/common/errors.py | 5 +- common/src/leap/soledad/common/l2db/__init__.py | 6 +- .../leap/soledad/common/l2db/backends/__init__.py | 16 +- .../leap/soledad/common/l2db/backends/inmemory.py | 11 +- .../soledad/common/l2db/backends/sqlite_backend.py | 16 +- .../src/leap/soledad/common/l2db/query_parser.py | 11 +- .../leap/soledad/common/l2db/remote/http_app.py | 21 ++- .../leap/soledad/common/l2db/remote/http_client.py | 16 +- .../soledad/common/l2db/remote/http_database.py | 14 +- .../leap/soledad/common/l2db/remote/http_errors.py | 11 +- .../leap/soledad/common/l2db/remote/http_target.py | 15 +- common/src/leap/soledad/common/l2db/sync.py | 9 +- .../src/leap/soledad/common/tests/server_state.py | 2 +- common/src/leap/soledad/common/tests/test_async.py | 2 - common/src/leap/soledad/common/tests/test_couch.py | 15 +- common/src/leap/soledad/common/tests/test_http.py | 2 +- .../leap/soledad/common/tests/test_http_client.py | 5 +- common/src/leap/soledad/common/tests/test_https.py | 15 +- .../src/leap/soledad/common/tests/test_server.py | 9 +- .../leap/soledad/common/tests/test_soledad_app.py | 4 - .../leap/soledad/common/tests/test_sqlcipher.py | 8 +- .../soledad/common/tests/test_sqlcipher_sync.py | 13 +- common/src/leap/soledad/common/tests/test_sync.py | 2 - .../leap/soledad/common/tests/test_sync_target.py | 36 ++-- .../soledad/common/tests/u1db_tests/__init__.py | 14 +- .../common/tests/u1db_tests/test_backends.py | 19 +- .../common/tests/u1db_tests/test_document.py | 10 +- .../common/tests/u1db_tests/test_http_client.py | 18 +- .../common/tests/u1db_tests/test_http_database.py | 9 +- .../soledad/common/tests/u1db_tests/test_https.py | 196 ++++++++++----------- .../soledad/common/tests/u1db_tests/test_open.py | 10 +- common/src/leap/soledad/common/tests/util.py | 12 +- server/pkg/requirements.pip | 7 - server/src/leap/soledad/server/__init__.py | 6 +- server/src/leap/soledad/server/auth.py | 6 +- server/src/leap/soledad/server/sync.py | 6 +- 52 files changed, 305 insertions(+), 364 deletions(-) diff --git a/client/pkg/requirements.pip b/client/pkg/requirements.pip index 42c0d0b1..9596470f 100644 --- a/client/pkg/requirements.pip +++ b/client/pkg/requirements.pip @@ -1,10 +1,5 @@ pysqlcipher>2.6.3 -u1db scrypt zope.proxy twisted -# XXX -- fix me! -# oauth is not strictly needed by us, but we need it until u1db adds it to its -# release as a dep. -oauth diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 33eae2c4..8c25243b 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -39,8 +39,7 @@ from itertools import chain from StringIO import StringIO from collections import defaultdict -from u1db.remote import http_client -from u1db.remote.ssl_match_hostname import match_hostname + from twisted.internet.defer import DeferredLock, returnValue, inlineCallbacks from zope.interface import implements @@ -50,6 +49,8 @@ from leap.common.plugins import collect_plugins from leap.soledad.common import SHARED_DB_NAME from leap.soledad.common import soledad_assert from leap.soledad.common import soledad_assert_type +from leap.soledad.common.l2db.remote import http_client +from leap.soledad.common.l2db.remote.ssl_match_hostname import match_hostname from leap.soledad.client import adbapi from leap.soledad.client import events as soledad_events diff --git a/client/src/leap/soledad/client/auth.py b/client/src/leap/soledad/client/auth.py index 6dfabeb4..78e9bf1b 100644 --- a/client/src/leap/soledad/client/auth.py +++ b/client/src/leap/soledad/client/auth.py @@ -22,7 +22,7 @@ they can do token-based auth requests to the Soledad server. """ import base64 -from u1db import errors +from leap.soledad.common.l2db import errors class TokenBasedAuth(object): diff --git a/client/src/leap/soledad/client/http_target/api.py b/client/src/leap/soledad/client/http_target/api.py index b19ce9ce..f8de9a15 100644 --- a/client/src/leap/soledad/client/http_target/api.py +++ b/client/src/leap/soledad/client/http_target/api.py @@ -20,13 +20,13 @@ import json import base64 from uuid import uuid4 -from u1db import SyncTarget from twisted.web.error import Error from twisted.internet import defer -from leap.soledad.common.errors import InvalidAuthTokenError from leap.soledad.client.http_target.support import readBody +from leap.soledad.common.errors import InvalidAuthTokenError +from leap.soledad.common.l2db import SyncTarget # we may want to collect statistics from the sync process diff --git a/client/src/leap/soledad/client/http_target/fetch.py b/client/src/leap/soledad/client/http_target/fetch.py index 9801c3d9..a3f70b02 100644 --- a/client/src/leap/soledad/client/http_target/fetch.py +++ b/client/src/leap/soledad/client/http_target/fetch.py @@ -16,15 +16,17 @@ # along with this program. If not, see . import logging import json -from u1db import errors -from u1db.remote import utils + from twisted.internet import defer -from leap.soledad.common.document import SoledadDocument + from leap.soledad.client.events import SOLEDAD_SYNC_RECEIVE_STATUS from leap.soledad.client.events import emit_async from leap.soledad.client.crypto import is_symmetrically_encrypted from leap.soledad.client.encdecpool import SyncDecrypterPool from leap.soledad.client.http_target.support import RequestBody +from leap.soledad.common.document import SoledadDocument +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db.remote import utils logger = logging.getLogger(__name__) diff --git a/client/src/leap/soledad/client/http_target/send.py b/client/src/leap/soledad/client/http_target/send.py index 89288779..13218acf 100644 --- a/client/src/leap/soledad/client/http_target/send.py +++ b/client/src/leap/soledad/client/http_target/send.py @@ -16,10 +16,13 @@ # along with this program. If not, see . import json import logging + from twisted.internet import defer + from leap.soledad.client.events import emit_async from leap.soledad.client.events import SOLEDAD_SYNC_SEND_STATUS from leap.soledad.client.http_target.support import RequestBody + logger = logging.getLogger(__name__) diff --git a/client/src/leap/soledad/client/http_target/support.py b/client/src/leap/soledad/client/http_target/support.py index 2625744c..d82fe346 100644 --- a/client/src/leap/soledad/client/http_target/support.py +++ b/client/src/leap/soledad/client/http_target/support.py @@ -16,14 +16,15 @@ # along with this program. If not, see . import warnings import json -from u1db import errors -from u1db.remote import http_errors + from twisted.internet import defer from twisted.web.client import _ReadBodyProtocol from twisted.web.client import PartialDownloadError from twisted.web._newclient import ResponseDone from twisted.web._newclient import PotentialDataLoss +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db.remote import http_errors # we want to make sure that HTTP errors will raise appropriate u1db errors, # that is, fire errbacks with the appropriate failures, in the context of diff --git a/client/src/leap/soledad/client/shared_db.py b/client/src/leap/soledad/client/shared_db.py index a1d95fbe..d43db045 100644 --- a/client/src/leap/soledad/client/shared_db.py +++ b/client/src/leap/soledad/client/shared_db.py @@ -17,7 +17,7 @@ """ A shared database for storing/retrieving encrypted key material. """ -from u1db.remote import http_database +from leap.soledad.common.l2db.remote import http_database from leap.soledad.client.auth import TokenBasedAuth diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index 99f5dad8..bf2a50f1 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -44,10 +44,6 @@ handled by Soledad should be created by SQLCipher >= 2.0. import logging import os import json -import u1db - -from u1db import errors as u1db_errors -from u1db.backends import sqlite_backend from hashlib import sha256 from functools import partial @@ -58,11 +54,14 @@ from twisted.internet import reactor from twisted.internet import defer from twisted.enterprise import adbapi +from leap.soledad.common.document import SoledadDocument +from leap.soledad.common import l2db +from leap.soledad.common.l2db import errors as u1db_errors +from leap.soledad.common.l2db.backends import sqlite_backend + from leap.soledad.client.http_target import SoledadHTTPSyncTarget from leap.soledad.client.sync import SoledadSynchronizer - from leap.soledad.client import pragmas -from leap.soledad.common.document import SoledadDocument logger = logging.getLogger(__name__) diff --git a/client/src/leap/soledad/client/sync.py b/client/src/leap/soledad/client/sync.py index 9cafe62f..2656a150 100644 --- a/client/src/leap/soledad/client/sync.py +++ b/client/src/leap/soledad/client/sync.py @@ -23,9 +23,9 @@ import logging from twisted.internet import defer -from u1db import errors +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db.sync import Synchronizer from leap.soledad.common.errors import BackendNotReadyError -from u1db.sync import Synchronizer logger = logging.getLogger(__name__) diff --git a/common/pkg/requirements.pip b/common/pkg/requirements.pip index a1238707..e69de29b 100644 --- a/common/pkg/requirements.pip +++ b/common/pkg/requirements.pip @@ -1,6 +0,0 @@ -u1db - -# XXX -- fix me! -# oauth is not strictly needed by us, but we need it until u1db adds it to its -# release as a dep. -oauth diff --git a/common/src/leap/soledad/common/README.txt b/common/src/leap/soledad/common/README.txt index 106efb5e..38b9858e 100644 --- a/common/src/leap/soledad/common/README.txt +++ b/common/src/leap/soledad/common/README.txt @@ -3,10 +3,10 @@ Soledad common package This package contains Soledad bits used by both server and client. -Couch U1DB Backend +Couch L2DB Backend ------------------ -U1DB backends rely on some atomic operations that modify documents contents +L2DB backends rely on some atomic operations that modify documents contents and metadata (conflicts, transaction ids and indexes). The only atomic operation in Couch is a document put, so every u1db atomic operation has to be mapped to a couch document put. diff --git a/common/src/leap/soledad/common/backend.py b/common/src/leap/soledad/common/backend.py index 0a36c068..f4f48f86 100644 --- a/common/src/leap/soledad/common/backend.py +++ b/common/src/leap/soledad/common/backend.py @@ -16,27 +16,28 @@ # along with this program. If not, see . -"""A U1DB generic backend.""" +"""A L2DB generic backend.""" import functools -from u1db import vectorclock -from u1db.errors import ( + +from leap.soledad.common.document import ServerDocument +from leap.soledad.common.l2db import vectorclock +from leap.soledad.common.l2db.errors import ( RevisionConflict, InvalidDocId, ConflictedDoc, DocumentDoesNotExist, DocumentAlreadyDeleted, ) -from u1db.backends import CommonBackend -from u1db.backends import CommonSyncTarget -from leap.soledad.common.document import ServerDocument +from leap.soledad.common.l2db.backends import CommonBackend +from leap.soledad.common.l2db.backends import CommonSyncTarget class SoledadBackend(CommonBackend): BATCH_SUPPORT = False """ - A U1DB backend implementation. + A L2DB backend implementation. """ def __init__(self, database, replica_uid=None): diff --git a/common/src/leap/soledad/common/couch/__init__.py b/common/src/leap/soledad/common/couch/__init__.py index 8c60b6a4..523a50a0 100644 --- a/common/src/leap/soledad/common/couch/__init__.py +++ b/common/src/leap/soledad/common/couch/__init__.py @@ -42,12 +42,12 @@ from couchdb.http import ( urljoin as couch_urljoin, Resource, ) -from u1db.errors import ( +from leap.soledad.common.l2db.errors import ( DatabaseDoesNotExist, InvalidGeneration, RevisionConflict, ) -from u1db.remote import http_app +from leap.soledad.common.l2db.remote import http_app from leap.soledad.common import ddocs diff --git a/common/src/leap/soledad/common/couch/state.py b/common/src/leap/soledad/common/couch/state.py index 4f07c105..9b40a264 100644 --- a/common/src/leap/soledad/common/couch/state.py +++ b/common/src/leap/soledad/common/couch/state.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # state.py -# Copyright (C) 2015 LEAP +# Copyright (C) 2015,2016 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,17 +17,17 @@ """ Server state using CouchDatabase as backend. """ -import re import logging +import re import time from urlparse import urljoin from hashlib import sha512 -from u1db.remote.server_state import ServerState -from leap.soledad.common.command import exec_validated_cmd from leap.soledad.common.couch import CouchDatabase from leap.soledad.common.couch import couch_server -from u1db.errors import Unauthorized +from leap.soledad.common.command import exec_validated_cmd +from leap.soledad.common.l2db.remote.server_state import ServerState +from leap.soledad.common.l2db.errors import Unauthorized logger = logging.getLogger(__name__) diff --git a/common/src/leap/soledad/common/document.py b/common/src/leap/soledad/common/document.py index 9e0c0976..6c26a29f 100644 --- a/common/src/leap/soledad/common/document.py +++ b/common/src/leap/soledad/common/document.py @@ -17,11 +17,11 @@ """ -A Soledad Document is an u1db.Document with lasers. +A Soledad Document is an l2db.Document with lasers. """ -from u1db import Document +from .l2db import Document # diff --git a/common/src/leap/soledad/common/errors.py b/common/src/leap/soledad/common/errors.py index 76a7240d..dec871c9 100644 --- a/common/src/leap/soledad/common/errors.py +++ b/common/src/leap/soledad/common/errors.py @@ -20,9 +20,8 @@ Soledad errors. """ - -from u1db import errors -from u1db.remote import http_errors +from .l2db import errors +from .l2db.remote import http_errors def register_exception(cls): diff --git a/common/src/leap/soledad/common/l2db/__init__.py b/common/src/leap/soledad/common/l2db/__init__.py index e33309a4..cc121d06 100644 --- a/common/src/leap/soledad/common/l2db/__init__.py +++ b/common/src/leap/soledad/common/l2db/__init__.py @@ -14,14 +14,14 @@ # You should have received a copy of the GNU Lesser General Public License # along with u1db. If not, see . -"""U1DB""" +"""L2DB""" try: import simplejson as json except ImportError: import json # noqa -from u1db.errors import InvalidJSON, InvalidContent +from leap.soledad.common.l2db.errors import InvalidJSON, InvalidContent __version_info__ = (13, 9) __version__ = '.'.join(map(lambda x: '%02d' % x, __version_info__)) @@ -40,7 +40,7 @@ def open(path, create, document_factory=None): parameters as Document.__init__. :return: An instance of Database. """ - from u1db.backends import sqlite_backend + from leap.soledad.common.l2db.backends import sqlite_backend return sqlite_backend.SQLiteDatabase.open_database( path, create=create, document_factory=document_factory) diff --git a/common/src/leap/soledad/common/l2db/backends/__init__.py b/common/src/leap/soledad/common/l2db/backends/__init__.py index a647c8aa..922daafd 100644 --- a/common/src/leap/soledad/common/l2db/backends/__init__.py +++ b/common/src/leap/soledad/common/l2db/backends/__init__.py @@ -23,22 +23,20 @@ except ImportError: import json # noqa import uuid -import u1db -from u1db import ( - errors, -) -import u1db.sync -from u1db.vectorclock import VectorClockRev +from leap.soledad.common import l2db +from leap.soledad.common.l2db import sync as l2db_sync +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db.vectorclock import VectorClockRev -check_doc_id_re = re.compile("^" + u1db.DOC_ID_CONSTRAINTS + "$", re.UNICODE) +check_doc_id_re = re.compile("^" + l2db.DOC_ID_CONSTRAINTS + "$", re.UNICODE) -class CommonSyncTarget(u1db.sync.LocalSyncTarget): +class CommonSyncTarget(l2db_sync.LocalSyncTarget): pass -class CommonBackend(u1db.Database): +class CommonBackend(l2db.Database): document_size_limit = 0 diff --git a/common/src/leap/soledad/common/l2db/backends/inmemory.py b/common/src/leap/soledad/common/l2db/backends/inmemory.py index 1feb1604..06a934a6 100644 --- a/common/src/leap/soledad/common/l2db/backends/inmemory.py +++ b/common/src/leap/soledad/common/l2db/backends/inmemory.py @@ -21,13 +21,10 @@ try: except ImportError: import json # noqa -from u1db import ( - Document, - errors, - query_parser, - vectorclock, - ) -from u1db.backends import CommonBackend, CommonSyncTarget +from leap.soledad.common.l2db import ( + Document, errors, + query_parser, vectorclock) +from leap.soledad.common.l2db.backends import CommonBackend, CommonSyncTarget def get_prefix(value): diff --git a/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py b/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py index 773213b5..309000ee 100644 --- a/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py +++ b/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py @@ -1,4 +1,5 @@ # Copyright 2011 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project # # This file is part of u1db. # @@ -14,7 +15,9 @@ # You should have received a copy of the GNU Lesser General Public License # along with u1db. If not, see . -"""A U1DB implementation that uses SQLite as its persistence layer.""" +""" +A L2DB implementation that uses SQLite as its persistence layer. +""" import errno import os @@ -29,13 +32,10 @@ import uuid import pkg_resources -from u1db.backends import CommonBackend, CommonSyncTarget -from u1db import ( - Document, - errors, - query_parser, - vectorclock, - ) +from leap.soledad.common.l2db.backends import CommonBackend, CommonSyncTarget +from leap.soledad.common.l2db import ( + Document, errors, + query_parser, vectorclock) class SQLiteDatabase(CommonBackend): diff --git a/common/src/leap/soledad/common/l2db/query_parser.py b/common/src/leap/soledad/common/l2db/query_parser.py index f564821f..7f07b554 100644 --- a/common/src/leap/soledad/common/l2db/query_parser.py +++ b/common/src/leap/soledad/common/l2db/query_parser.py @@ -1,4 +1,5 @@ # Copyright 2011 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project # # This file is part of u1db. # @@ -13,13 +14,13 @@ # # You should have received a copy of the GNU Lesser General Public License # along with u1db. If not, see . - -"""Code for parsing Index definitions.""" +""" +Code for parsing Index definitions. +""" import re -from u1db import ( - errors, - ) + +from leap.soledad.common.l2db import errors class Getter(object): diff --git a/common/src/leap/soledad/common/l2db/remote/http_app.py b/common/src/leap/soledad/common/l2db/remote/http_app.py index 85cdb029..3b65f5f7 100644 --- a/common/src/leap/soledad/common/l2db/remote/http_app.py +++ b/common/src/leap/soledad/common/l2db/remote/http_app.py @@ -1,4 +1,5 @@ # Copyright 2011-2012 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project # # This file is part of u1db. # @@ -14,7 +15,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with u1db. If not, see . -"""HTTP Application exposing U1DB.""" +""" +HTTP Application exposing U1DB. +""" + +# TODO -- deprecate, use twisted/txaio. import functools import httplib @@ -28,17 +33,11 @@ import urlparse import routes.mapper -from u1db import ( +from leap.soledad.common.l2db import ( __version__ as _u1db_version, - DBNAME_CONSTRAINTS, - Document, - errors, - sync, - ) -from u1db.remote import ( - http_errors, - utils, - ) + DBNAME_CONSTRAINTS, Document, + errors, sync) +from leap.soledad.common.l2db.remote import http_errors, utils def parse_bool(expression): diff --git a/common/src/leap/soledad/common/l2db/remote/http_client.py b/common/src/leap/soledad/common/l2db/remote/http_client.py index 2044d756..eea42888 100644 --- a/common/src/leap/soledad/common/l2db/remote/http_client.py +++ b/common/src/leap/soledad/common/l2db/remote/http_client.py @@ -1,4 +1,5 @@ # Copyright 2011-2012 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project # # This file is part of u1db. # @@ -29,17 +30,12 @@ import urlparse import urllib from time import sleep -from u1db import ( - errors, - ) -from u1db.remote import ( - http_errors, - ) - -from u1db.remote.ssl_match_hostname import ( # noqa +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db.remote import http_errors + +from leap.soledad.common.l2db.remote.ssl_match_hostname import ( # noqa CertificateError, - match_hostname, - ) + match_hostname) # Ubuntu/debian # XXX other... diff --git a/common/src/leap/soledad/common/l2db/remote/http_database.py b/common/src/leap/soledad/common/l2db/remote/http_database.py index 400e4020..d8dcfd55 100644 --- a/common/src/leap/soledad/common/l2db/remote/http_database.py +++ b/common/src/leap/soledad/common/l2db/remote/http_database.py @@ -22,16 +22,14 @@ except ImportError: import json # noqa import uuid -from u1db import ( +from leap.soledad.common.l2db import ( Database, Document, - errors, - ) -from u1db.remote import ( + errors) +from leap.soledad.common.l2db.remote import ( http_client, http_errors, - http_target, - ) + http_target) DOCUMENT_DELETED_STATUS = http_errors.wire_description_to_status[ @@ -152,8 +150,8 @@ class HTTPDatabase(http_client.HTTPClientBase, Database): if doc.doc_id is None: raise errors.InvalidDocId() params = {'old_rev': doc.rev} - res, headers = self._request_json('DELETE', - ['doc', doc.doc_id], params) + res, headers = self._request_json( + 'DELETE', ['doc', doc.doc_id], params) doc.make_tombstone() doc.rev = res['rev'] diff --git a/common/src/leap/soledad/common/l2db/remote/http_errors.py b/common/src/leap/soledad/common/l2db/remote/http_errors.py index f6bd01d3..ee4cfefa 100644 --- a/common/src/leap/soledad/common/l2db/remote/http_errors.py +++ b/common/src/leap/soledad/common/l2db/remote/http_errors.py @@ -1,4 +1,5 @@ # Copyright 2011-2012 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project # # This file is part of u1db. # @@ -14,11 +15,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with u1db. If not, see . -"""Information about the encoding of errors over HTTP.""" +""" +Information about the encoding of errors over HTTP. +""" -from u1db import ( - errors, - ) +from leap.soledad.common.l2db import errors # error wire descriptions mapping to HTTP status codes @@ -37,7 +38,7 @@ wire_description_to_status = dict([ (errors.InvalidReplicaUID.wire_description, 409), (errors.InvalidTransactionId.wire_description, 409), (errors.Unavailable.wire_description, 503), -# without matching exception + # without matching exception (errors.DOCUMENT_DELETED, 404) ]) diff --git a/common/src/leap/soledad/common/l2db/remote/http_target.py b/common/src/leap/soledad/common/l2db/remote/http_target.py index 1028963e..598170e4 100644 --- a/common/src/leap/soledad/common/l2db/remote/http_target.py +++ b/common/src/leap/soledad/common/l2db/remote/http_target.py @@ -21,17 +21,10 @@ try: except ImportError: import json # noqa -from u1db import ( - Document, - SyncTarget, - ) -from u1db.errors import ( - BrokenSyncStream, - ) -from u1db.remote import ( - http_client, - utils, - ) +from leap.soledad.common.l2db import Document, SyncTarget +from leap.soledad.common.l2db.errors import BrokenSyncStream +from leap.soledad.common.l2db.remote import ( + http_client, utils) class HTTPSyncTarget(http_client.HTTPClientBase, SyncTarget): diff --git a/common/src/leap/soledad/common/l2db/sync.py b/common/src/leap/soledad/common/l2db/sync.py index d9e455d8..26e67140 100644 --- a/common/src/leap/soledad/common/l2db/sync.py +++ b/common/src/leap/soledad/common/l2db/sync.py @@ -17,8 +17,8 @@ """The synchronization utilities for U1DB.""" from itertools import izip -import u1db -from u1db import errors +from leap.soledad.common import l2db +from leap.soledad.common.l2db import errors class Synchronizer(object): @@ -275,11 +275,10 @@ class SyncExchange(object): # for tests self._db._last_exchange_log['return'] = { 'docs': _outgoing_trace, - 'last_gen': self.new_gen - } + 'last_gen': self.new_gen} -class LocalSyncTarget(u1db.SyncTarget): +class LocalSyncTarget(l2db.SyncTarget): """Common sync target implementation logic for all local sync targets.""" def __init__(self, db): diff --git a/common/src/leap/soledad/common/tests/server_state.py b/common/src/leap/soledad/common/tests/server_state.py index 2fe9472f..26838f89 100644 --- a/common/src/leap/soledad/common/tests/server_state.py +++ b/common/src/leap/soledad/common/tests/server_state.py @@ -26,7 +26,7 @@ import errno import tempfile -from u1db.remote.server_state import ServerState +from leap.soledad.common.l2db.remote.server_state import ServerState from leap.soledad.common.tests.util import ( copy_sqlcipher_database_for_test, ) diff --git a/common/src/leap/soledad/common/tests/test_async.py b/common/src/leap/soledad/common/tests/test_async.py index 302ecc37..52be4ff3 100644 --- a/common/src/leap/soledad/common/tests/test_async.py +++ b/common/src/leap/soledad/common/tests/test_async.py @@ -14,8 +14,6 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - - import os import hashlib diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py index 7ba50e11..eefefc5d 100644 --- a/common/src/leap/soledad/common/tests/test_couch.py +++ b/common/src/leap/soledad/common/tests/test_couch.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # test_couch.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013-2016 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,26 +14,23 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - - """ Test ObjectStore and Couch backend bits. """ - - import json +from uuid import uuid4 from urlparse import urljoin + from couchdb.client import Server -from uuid import uuid4 from testscenarios import TestWithScenarios from twisted.trial import unittest from mock import Mock -from u1db import errors as u1db_errors -from u1db import SyncTarget -from u1db import vectorclock +from leap.soledad.common.l2db import errors as u1db_errors +from leap.soledad.common.l2db import SyncTarget +from leap.soledad.common.l2db import vectorclock from leap.soledad.common import couch from leap.soledad.common.document import ServerDocument diff --git a/common/src/leap/soledad/common/tests/test_http.py b/common/src/leap/soledad/common/tests/test_http.py index bc486fe3..2351748d 100644 --- a/common/src/leap/soledad/common/tests/test_http.py +++ b/common/src/leap/soledad/common/tests/test_http.py @@ -17,10 +17,10 @@ """ Test Leap backend bits: test http database """ -from u1db.remote import http_database from leap.soledad.client import auth from leap.soledad.common.tests.u1db_tests import test_http_database +from leap.soledad.common.l2db.remote import http_database # ----------------------------------------------------------------------------- diff --git a/common/src/leap/soledad/common/tests/test_http_client.py b/common/src/leap/soledad/common/tests/test_http_client.py index 700ae3b6..d932b2b0 100644 --- a/common/src/leap/soledad/common/tests/test_http_client.py +++ b/common/src/leap/soledad/common/tests/test_http_client.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # test_http_client.py -# Copyright (C) 2013, 2014 LEAP +# Copyright (C) 2013-2016 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,11 +19,10 @@ Test Leap backend bits: sync target """ import json -from u1db.remote import http_client - from testscenarios import TestWithScenarios from leap.soledad.client import auth +from leap.soledad.common.l2db.remote import http_client from leap.soledad.common.tests.u1db_tests import test_http_client from leap.soledad.server.auth import SoledadTokenAuthMiddleware diff --git a/common/src/leap/soledad/common/tests/test_https.py b/common/src/leap/soledad/common/tests/test_https.py index eeeb4982..8d9b8d92 100644 --- a/common/src/leap/soledad/common/tests/test_https.py +++ b/common/src/leap/soledad/common/tests/test_https.py @@ -14,19 +14,16 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - - """ Test Leap backend bits: https """ +from unittest import skip - -from u1db.remote import http_client +from testscenarios import TestWithScenarios from leap.soledad import client -from testscenarios import TestWithScenarios - +from leap.soledad.common.l2db.remote import http_client from leap.soledad.common.tests.u1db_tests import test_backends from leap.soledad.common.tests.u1db_tests import test_https from leap.soledad.common.tests.util import ( @@ -65,14 +62,16 @@ def token_leap_https_sync_target(test, host, path, cert_file=None): return st +@skip("Skiping tests imported from U1DB.") class TestSoledadHTTPSyncTargetHttpsSupport( TestWithScenarios, - test_https.TestHttpSyncTargetHttpsSupport, + # test_https.TestHttpSyncTargetHttpsSupport, BaseSoledadTest): scenarios = [ ('token_soledad_https', - {'server_def': test_https.https_server_def, + { + #'server_def': test_https.https_server_def, 'make_app_with_state': make_token_soledad_app, 'make_document_for_test': make_soledad_document_for_test, 'sync_target': token_leap_https_sync_target}), diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py index 2fee119d..357027e9 100644 --- a/common/src/leap/soledad/common/tests/test_server.py +++ b/common/src/leap/soledad/common/tests/test_server.py @@ -17,16 +17,17 @@ """ Tests for server-related functionality. """ +import binascii +import mock import os import tempfile -import mock import time -import binascii + +from hashlib import sha512 from pkg_resources import resource_filename +from urlparse import urljoin from uuid import uuid4 -from hashlib import sha512 -from urlparse import urljoin from twisted.internet import defer from twisted.trial import unittest diff --git a/common/src/leap/soledad/common/tests/test_soledad_app.py b/common/src/leap/soledad/common/tests/test_soledad_app.py index 4598a7bb..7f9a58d3 100644 --- a/common/src/leap/soledad/common/tests/test_soledad_app.py +++ b/common/src/leap/soledad/common/tests/test_soledad_app.py @@ -14,13 +14,9 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - - """ Test ObjectStore and Couch backend bits. """ - - from testscenarios import TestWithScenarios from leap.soledad.common.tests.util import BaseSoledadTest diff --git a/common/src/leap/soledad/common/tests/test_sqlcipher.py b/common/src/leap/soledad/common/tests/test_sqlcipher.py index 8105c56e..2bcdf0fb 100644 --- a/common/src/leap/soledad/common/tests/test_sqlcipher.py +++ b/common/src/leap/soledad/common/tests/test_sqlcipher.py @@ -26,10 +26,10 @@ import shutil from pysqlcipher import dbapi2 from testscenarios import TestWithScenarios -# u1db stuff. -from u1db import errors -from u1db import query_parser -from u1db.backends.sqlite_backend import SQLitePartialExpandDatabase +# l2db stuff. +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db import query_parser +from leap.soledad.common.l2db.backends.sqlite_backend import SQLitePartialExpandDatabase # soledad stuff. from leap.soledad.common import soledad_assert diff --git a/common/src/leap/soledad/common/tests/test_sqlcipher_sync.py b/common/src/leap/soledad/common/tests/test_sqlcipher_sync.py index 439fc070..42cfa6b7 100644 --- a/common/src/leap/soledad/common/tests/test_sqlcipher_sync.py +++ b/common/src/leap/soledad/common/tests/test_sqlcipher_sync.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # test_sqlcipher.py -# Copyright (C) 2013 LEAP +# Copyright (C) 2013-2016 LEAP # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,20 +17,19 @@ """ Test sqlcipher backend sync. """ - - import os -from u1db import sync -from u1db import vectorclock -from u1db import errors from uuid import uuid4 from testscenarios import TestWithScenarios +from leap.soledad.common.l2db import sync +from leap.soledad.common.l2db import vectorclock +from leap.soledad.common.l2db import errors + from leap.soledad.common.crypto import ENC_SCHEME_KEY -from leap.soledad.client.http_target import SoledadHTTPSyncTarget from leap.soledad.client.crypto import decrypt_doc_dict +from leap.soledad.client.http_target import SoledadHTTPSyncTarget from leap.soledad.common.tests import u1db_tests as tests from leap.soledad.common.tests.test_sqlcipher import SQLCIPHER_SCENARIOS diff --git a/common/src/leap/soledad/common/tests/test_sync.py b/common/src/leap/soledad/common/tests/test_sync.py index 1041367b..cc18d387 100644 --- a/common/src/leap/soledad/common/tests/test_sync.py +++ b/common/src/leap/soledad/common/tests/test_sync.py @@ -14,8 +14,6 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - - import json import tempfile import threading diff --git a/common/src/leap/soledad/common/tests/test_sync_target.py b/common/src/leap/soledad/common/tests/test_sync_target.py index f25e84dd..c9b705a3 100644 --- a/common/src/leap/soledad/common/tests/test_sync_target.py +++ b/common/src/leap/soledad/common/tests/test_sync_target.py @@ -21,7 +21,6 @@ import cStringIO import os import time import json -import u1db import random import string import shutil @@ -36,8 +35,9 @@ from leap.soledad.client.sqlcipher import SQLCipherU1DBSync from leap.soledad.client.sqlcipher import SQLCipherOptions from leap.soledad.client.sqlcipher import SQLCipherDatabase -from leap.soledad.common.document import SoledadDocument +from leap.soledad.common import l2db +from leap.soledad.common.document import SoledadDocument from leap.soledad.common.tests import u1db_tests as tests from leap.soledad.common.tests.util import make_sqlcipher_database_for_test from leap.soledad.common.tests.util import make_soledad_app @@ -90,53 +90,53 @@ class TestSoledadParseReceivedDocResponse(SoledadWithCouchServerMixin): doc.get_json(), doc.doc_id, doc.rev, key, secret) - with self.assertRaises(u1db.errors.BrokenSyncStream): + with self.assertRaises(l2db.errors.BrokenSyncStream): self.target._parse_received_doc_response("[\r\n{},\r\n]") - with self.assertRaises(u1db.errors.BrokenSyncStream): + with self.assertRaises(l2db.errors.BrokenSyncStream): self.target._parse_received_doc_response( ('[\r\n{},\r\n{"id": "i", "rev": "r", ' + '"content": %s, "gen": 3, "trans_id": "T-sid"}' + ',\r\n]') % json.dumps(enc_json)) def test_wrong_start(self): - with self.assertRaises(u1db.errors.BrokenSyncStream): + with self.assertRaises(l2db.errors.BrokenSyncStream): self.target._parse_received_doc_response("{}\r\n]") - with self.assertRaises(u1db.errors.BrokenSyncStream): + with self.assertRaises(l2db.errors.BrokenSyncStream): self.target._parse_received_doc_response("\r\n{}\r\n]") - with self.assertRaises(u1db.errors.BrokenSyncStream): + with self.assertRaises(l2db.errors.BrokenSyncStream): self.target._parse_received_doc_response("") def test_wrong_end(self): - with self.assertRaises(u1db.errors.BrokenSyncStream): + with self.assertRaises(l2db.errors.BrokenSyncStream): self.target._parse_received_doc_response("[\r\n{}") - with self.assertRaises(u1db.errors.BrokenSyncStream): + with self.assertRaises(l2db.errors.BrokenSyncStream): self.target._parse_received_doc_response("[\r\n") def test_missing_comma(self): - with self.assertRaises(u1db.errors.BrokenSyncStream): + with self.assertRaises(l2db.errors.BrokenSyncStream): self.target._parse_received_doc_response( '[\r\n{}\r\n{"id": "i", "rev": "r", ' '"content": "c", "gen": 3}\r\n]') def test_no_entries(self): - with self.assertRaises(u1db.errors.BrokenSyncStream): + with self.assertRaises(l2db.errors.BrokenSyncStream): self.target._parse_received_doc_response("[\r\n]") def test_error_in_stream(self): - with self.assertRaises(u1db.errors.BrokenSyncStream): + with self.assertRaises(l2db.errors.BrokenSyncStream): self.target._parse_received_doc_response( '[\r\n{"new_generation": 0},' '\r\n{"error": "unavailable"}\r\n') - with self.assertRaises(u1db.errors.BrokenSyncStream): + with self.assertRaises(l2db.errors.BrokenSyncStream): self.target._parse_received_doc_response( '[\r\n{"error": "unavailable"}\r\n') - with self.assertRaises(u1db.errors.BrokenSyncStream): + with self.assertRaises(l2db.errors.BrokenSyncStream): self.target._parse_received_doc_response('[\r\n{"error": "?"}\r\n') # @@ -256,7 +256,7 @@ class TestSoledadSyncTarget( replica_trans_id=None, number_of_docs=None, doc_idx=None, sync_id=None): if doc.doc_id in trigger_ids: - raise u1db.errors.U1DBError + raise l2db.errors.U1DBError return _put_doc_if_newer(doc, save_conflict=save_conflict, replica_uid=replica_uid, replica_gen=replica_gen, @@ -278,7 +278,7 @@ class TestSoledadSyncTarget( doc2 = self.make_document('doc-here2', 'replica:1', '{"value": "here2"}') - with self.assertRaises(u1db.errors.U1DBError): + with self.assertRaises(l2db.errors.U1DBError): yield remote_target.sync_exchange( [(doc1, 10, 'T-sid'), (doc2, 11, 'T-sud')], 'replica', @@ -706,7 +706,7 @@ class SoledadDatabaseSyncTargetTests( def before_get_docs_explode(state): if state != 'before get_docs': return - raise u1db.errors.U1DBError("fail") + raise l2db.errors.U1DBError("fail") self.set_trace_hook(before_get_docs_explode) # suppress traceback printing in the wsgiref server # self.patch(simple_server.ServerHandler, @@ -714,7 +714,7 @@ class SoledadDatabaseSyncTargetTests( doc = self.db.create_doc_from_json(tests.simple_doc) self.assertTransactionLog([doc.doc_id], self.db) self.assertRaises( - (u1db.errors.U1DBError, u1db.errors.BrokenSyncStream), + (l2db.errors.U1DBError, l2db.errors.BrokenSyncStream), self.st.sync_exchange, [], 'other-replica', last_known_generation=0, last_known_trans_id=None, insert_doc_cb=self.receive_doc) diff --git a/common/src/leap/soledad/common/tests/u1db_tests/__init__.py b/common/src/leap/soledad/common/tests/u1db_tests/__init__.py index 01da9381..7f334b4a 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/__init__.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/__init__.py @@ -36,13 +36,13 @@ from twisted.web.server import Site from twisted.web.wsgi import WSGIResource from twisted.internet import reactor -from u1db import errors -from u1db import Document -from u1db.backends import inmemory -from u1db.backends import sqlite_backend -from u1db.remote import server_state -from u1db.remote import http_app -from u1db.remote import http_target +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db import Document +from leap.soledad.common.l2db.backends import inmemory +from leap.soledad.common.l2db.backends import sqlite_backend +from leap.soledad.common.l2db.remote import server_state +from leap.soledad.common.l2db.remote import http_app +from leap.soledad.common.l2db.remote import http_target class TestCase(unittest.TestCase): diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py b/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py index 410d838f..c0c6ea6b 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py @@ -1,8 +1,9 @@ # Copyright 2011 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project # -# This file is part of u1db. +# This file is part of leap.soledad.common # -# u1db is free software: you can redistribute it and/or modify +# leap.soledad.common is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License version 3 # as published by the Free Software Foundation. # @@ -14,21 +15,21 @@ # You should have received a copy of the GNU Lesser General Public License # along with u1db. If not, see . -"""The backend class for U1DB. This deals with hiding storage details.""" +""" +The backend class for L2DB. This deals with hiding storage details. +""" import json -from u1db import DocumentBase -from u1db import errors -from u1db import vectorclock +from leap.soledad.common.l2db import DocumentBase +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db import vectorclock +from leap.soledad.common.l2db.remote import http_database from leap.soledad.common.tests import u1db_tests as tests - from leap.soledad.common.tests.u1db_tests import make_http_app from leap.soledad.common.tests.u1db_tests import make_oauth_http_app -from u1db.remote import http_database - from unittest import skip simple_doc = tests.simple_doc diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_document.py b/common/src/leap/soledad/common/tests/u1db_tests/test_document.py index 23502b4b..4e8bcaf9 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_document.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_document.py @@ -1,8 +1,9 @@ # Copyright 2011 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project # -# This file is part of u1db. +# This file is part of leap.soledad.common # -# u1db is free software: you can redistribute it and/or modify +# leap.soledad.common is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License version 3 # as published by the Free Software Foundation. # @@ -13,10 +14,9 @@ # # You should have received a copy of the GNU Lesser General Public License # along with u1db. If not, see . - - from unittest import skip -from u1db import errors + +from leap.soledad.common.l2db import errors from leap.soledad.common.tests import u1db_tests as tests diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py b/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py index 973c3b26..344dcb29 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py @@ -1,4 +1,5 @@ # Copyright 2011-2012 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project # # This file is part of u1db. # @@ -14,22 +15,17 @@ # You should have received a copy of the GNU Lesser General Public License # along with u1db. If not, see . -"""Tests for HTTPDatabase""" - -from oauth import oauth +""" +Tests for HTTPDatabase +""" import json -from u1db import ( - errors, -) - from unittest import skip -from leap.soledad.common.tests import u1db_tests as tests +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db.remote import http_client -from u1db.remote import ( - http_client, -) +from leap.soledad.common.tests import u1db_tests as tests @skip("Skiping tests imported from U1DB.") diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py b/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py index 015e6e69..001aebd4 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py @@ -21,11 +21,10 @@ import json from unittest import skip -from u1db import errors -from u1db import Document -from u1db.remote import http_database -from u1db.remote import http_target - +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db import Document +from leap.soledad.common.l2db.remote import http_database +from leap.soledad.common.l2db.remote import http_target from leap.soledad.common.tests import u1db_tests as tests from leap.soledad.common.tests.u1db_tests import make_http_app diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_https.py b/common/src/leap/soledad/common/tests/u1db_tests/test_https.py index e177a808..8a5743e7 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_https.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_https.py @@ -4,119 +4,111 @@ import os import ssl import sys -from paste import httpserver +#from paste import httpserver from unittest import skip -from u1db.remote import http_client -from u1db.remote import http_target +from leap.soledad.common.l2db.remote import http_client +#from leap.soledad.common.l2db.remote import http_target from leap import soledad from leap.soledad.common.tests import u1db_tests as tests from leap.soledad.common.tests.u1db_tests import make_oauth_http_app -def https_server_def(): - def make_server(host_port, application): - from OpenSSL import SSL - cert_file = os.path.join(os.path.dirname(__file__), 'testing-certs', - 'testing.cert') - key_file = os.path.join(os.path.dirname(__file__), 'testing-certs', - 'testing.key') - ssl_context = SSL.Context(SSL.SSLv23_METHOD) - ssl_context.use_privatekey_file(key_file) - ssl_context.use_certificate_chain_file(cert_file) - srv = httpserver.WSGIServerBase(application, host_port, - httpserver.WSGIHandler, - ssl_context=ssl_context - ) - - def shutdown_request(req): - req.shutdown() - srv.close_request(req) - - srv.shutdown_request = shutdown_request - application.base_url = "https://localhost:%s" % srv.server_address[1] - return srv - return make_server, "shutdown", "https" - - -def oauth_https_sync_target(test, host, path): - _, port = test.server.server_address - st = http_target.HTTPSyncTarget('https://%s:%d/~/%s' % (host, port, path)) - st.set_oauth_credentials(tests.consumer1.key, tests.consumer1.secret, - tests.token1.key, tests.token1.secret) - return st - - -@skip("Skiping tests imported from U1DB.") -class TestHttpSyncTargetHttpsSupport(tests.TestCaseWithServer): - - scenarios = [ - ('oauth_https', {'server_def': https_server_def, - 'make_app_with_state': make_oauth_http_app, - 'make_document_for_test': - tests.make_document_for_test, - 'sync_target': oauth_https_sync_target - }), - ] - - def setUp(self): - try: - import OpenSSL # noqa - except ImportError: - self.skipTest("Requires pyOpenSSL") - self.cacert_pem = os.path.join(os.path.dirname(__file__), - 'testing-certs', 'cacert.pem') +#def https_server_def(): + #def make_server(host_port, application): + #from OpenSSL import SSL + #cert_file = os.path.join(os.path.dirname(__file__), 'testing-certs', + #'testing.cert') + #key_file = os.path.join(os.path.dirname(__file__), 'testing-certs', + #'testing.key') + #ssl_context = SSL.Context(SSL.SSLv23_METHOD) + #ssl_context.use_privatekey_file(key_file) + #ssl_context.use_certificate_chain_file(cert_file) + #srv = httpserver.WSGIServerBase(application, host_port, + #httpserver.WSGIHandler, + #ssl_context=ssl_context + #) +# + #def shutdown_request(req): + #req.shutdown() + #srv.close_request(req) +# + #srv.shutdown_request = shutdown_request + #application.base_url = "https://localhost:%s" % srv.server_address[1] + #return srv + #return make_server, "shutdown", "https" + + +#@skip("Skiping tests imported from U1DB.") +#class TestHttpSyncTargetHttpsSupport(tests.TestCaseWithServer): +# + #scenarios = [ + #('oauth_https', {'server_def': https_server_def, + #'make_app_with_state': make_oauth_http_app, + #'make_document_for_test': + #tests.make_document_for_test, + #'sync_target': oauth_https_sync_target + #}), + #] +# + #def setUp(self): + #try: + #import OpenSSL # noqa + #except ImportError: + #self.skipTest("Requires pyOpenSSL") + #self.cacert_pem = os.path.join(os.path.dirname(__file__), + #'testing-certs', 'cacert.pem') # The default u1db http_client class for doing HTTPS only does HTTPS # if the platform is linux. Because of this, soledad replaces that # class with one that will do HTTPS independent of the platform. In # order to maintain the compatibility with u1db default tests, we undo # that replacement here. - http_client._VerifiedHTTPSConnection = \ - soledad.client.api.old__VerifiedHTTPSConnection - super(TestHttpSyncTargetHttpsSupport, self).setUp() - - def getSyncTarget(self, host, path=None, cert_file=None): - if self.server is None: - self.startServer() - return self.sync_target(self, host, path, cert_file=cert_file) - - def test_working(self): - self.startServer() - db = self.request_state._create_database('test') - self.patch(http_client, 'CA_CERTS', self.cacert_pem) - remote_target = self.getSyncTarget('localhost', 'test') - remote_target.record_sync_info('other-id', 2, 'T-id') - self.assertEqual( - (2, 'T-id'), db._get_replica_gen_and_trans_id('other-id')) - - def test_cannot_verify_cert(self): - if not sys.platform.startswith('linux'): - self.skipTest( - "XXX certificate verification happens on linux only for now") - self.startServer() + #http_client._VerifiedHTTPSConnection = \ + #soledad.client.api.old__VerifiedHTTPSConnection + #super(TestHttpSyncTargetHttpsSupport, self).setUp() +# + #def getSyncTarget(self, host, path=None, cert_file=None): + #if self.server is None: + #self.startServer() + #return self.sync_target(self, host, path, cert_file=cert_file) +# + #def test_working(self): + #self.startServer() + #db = self.request_state._create_database('test') + #self.patch(http_client, 'CA_CERTS', self.cacert_pem) + #remote_target = self.getSyncTarget('localhost', 'test') + #remote_target.record_sync_info('other-id', 2, 'T-id') + #self.assertEqual( + #(2, 'T-id'), db._get_replica_gen_and_trans_id('other-id')) +# + #def test_cannot_verify_cert(self): + #if not sys.platform.startswith('linux'): + #self.skipTest( + #"XXX certificate verification happens on linux only for now") + #self.startServer() # don't print expected traceback server-side - self.server.handle_error = lambda req, cli_addr: None - self.request_state._create_database('test') - remote_target = self.getSyncTarget('localhost', 'test') - try: - remote_target.record_sync_info('other-id', 2, 'T-id') - except ssl.SSLError, e: - self.assertIn("certificate verify failed", str(e)) - else: - self.fail("certificate verification should have failed.") - - def test_host_mismatch(self): - if not sys.platform.startswith('linux'): - self.skipTest( - "XXX certificate verification happens on linux only for now") - self.startServer() - self.request_state._create_database('test') - self.patch(http_client, 'CA_CERTS', self.cacert_pem) - remote_target = self.getSyncTarget('127.0.0.1', 'test') - self.assertRaises( - http_client.CertificateError, remote_target.record_sync_info, - 'other-id', 2, 'T-id') - - -load_tests = tests.load_with_scenarios + #self.server.handle_error = lambda req, cli_addr: None + #self.request_state._create_database('test') + #remote_target = self.getSyncTarget('localhost', 'test') + #try: + #remote_target.record_sync_info('other-id', 2, 'T-id') + #except ssl.SSLError, e: + #self.assertIn("certificate verify failed", str(e)) + #else: + #self.fail("certificate verification should have failed.") +# + #def test_host_mismatch(self): + #if not sys.platform.startswith('linux'): + #self.skipTest( + #"XXX certificate verification happens on linux only for now") + #self.startServer() + #self.request_state._create_database('test') + #self.patch(http_client, 'CA_CERTS', self.cacert_pem) + #remote_target = self.getSyncTarget('127.0.0.1', 'test') + #self.assertRaises( + #http_client.CertificateError, remote_target.record_sync_info, + #'other-id', 2, 'T-id') +# +# +#load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_open.py b/common/src/leap/soledad/common/tests/u1db_tests/test_open.py index ee249e6e..2fc04e38 100644 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_open.py +++ b/common/src/leap/soledad/common/tests/u1db_tests/test_open.py @@ -1,4 +1,5 @@ # Copyright 2011 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project # # This file is part of u1db. # @@ -17,14 +18,13 @@ """Test u1db.open""" import os +from unittest import skip -from u1db import ( - errors, - open as u1db_open, +from leap.soledad.common.l2db import ( + errors, open as u1db_open, ) -from unittest import skip from leap.soledad.common.tests import u1db_tests as tests -from u1db.backends import sqlite_backend +from leap.soledad.common.l2db.backends import sqlite_backend from leap.soledad.common.tests.u1db_tests.test_backends \ import TestAlternativeDocument diff --git a/common/src/leap/soledad/common/tests/util.py b/common/src/leap/soledad/common/tests/util.py index c681fa93..abe531ce 100644 --- a/common/src/leap/soledad/common/tests/util.py +++ b/common/src/leap/soledad/common/tests/util.py @@ -26,7 +26,6 @@ import tempfile import shutil import random import string -import u1db import couchdb from uuid import uuid4 @@ -35,13 +34,14 @@ from urlparse import urljoin from StringIO import StringIO from pysqlcipher import dbapi2 -from u1db import sync -from u1db.remote import http_database - from twisted.trial import unittest from leap.common.testing.basetest import BaseLeapTest +from leap.soledad.common import l2db +from leap.soledad.common.l2db import sync +from leap.soledad.common.l2db.remote import http_database + from leap.soledad.common import soledad_assert from leap.soledad.common.document import SoledadDocument from leap.soledad.common.couch import CouchDatabase @@ -234,9 +234,9 @@ class BaseSoledadTest(BaseLeapTest, MockedSharedDBTest): self.db2_file = os.path.join(self.tempdir, "db2.u1db") self.email = ADDRESS # open test dbs - self._db1 = u1db.open(self.db1_file, create=True, + self._db1 = l2db.open(self.db1_file, create=True, document_factory=SoledadDocument) - self._db2 = u1db.open(self.db2_file, create=True, + self._db2 = l2db.open(self.db2_file, create=True, document_factory=SoledadDocument) # get a random prefix for each test, so we do not mess with # concurrency during initialization and shutting down of diff --git a/server/pkg/requirements.pip b/server/pkg/requirements.pip index f9cce08e..2d845f24 100644 --- a/server/pkg/requirements.pip +++ b/server/pkg/requirements.pip @@ -1,13 +1,6 @@ configparser -u1db -routes PyOpenSSL twisted>=12.3.0 #pinned for wheezy compatibility Beaker==1.6.3 #wheezy couchdb==0.8 #wheezy - -# XXX -- fix me! -# oauth is not strictly needed by us, but we need it until u1db adds it to its -# release as a dep. -oauth diff --git a/server/src/leap/soledad/server/__init__.py b/server/src/leap/soledad/server/__init__.py index 195714c1..34570b52 100644 --- a/server/src/leap/soledad/server/__init__.py +++ b/server/src/leap/soledad/server/__init__.py @@ -84,9 +84,7 @@ import configparser import urlparse import sys -from u1db.remote import http_app, utils - -from ._version import get_versions +from leap.soledad.common.l2db.remote import http_app, utils from leap.soledad.server.auth import SoledadTokenAuthMiddleware from leap.soledad.server.gzip_middleware import GzipMiddleware @@ -100,6 +98,8 @@ from leap.soledad.common import SHARED_DB_NAME from leap.soledad.common.backend import SoledadBackend from leap.soledad.common.couch.state import CouchServerState +from ._version import get_versions + # ---------------------------------------------------------------------------- # Soledad WSGI application # ---------------------------------------------------------------------------- diff --git a/server/src/leap/soledad/server/auth.py b/server/src/leap/soledad/server/auth.py index 0ce1f497..ecee2d5d 100644 --- a/server/src/leap/soledad/server/auth.py +++ b/server/src/leap/soledad/server/auth.py @@ -14,21 +14,17 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . - - """ Authentication facilities for Soledad Server. """ - - import httplib import json -from u1db import DBNAME_CONSTRAINTS, errors as u1db_errors from abc import ABCMeta, abstractmethod from routes.mapper import Mapper from twisted.python import log +from leap.soledad.common.l2db import DBNAME_CONSTRAINTS, errors as u1db_errors from leap.soledad.common import SHARED_DB_NAME from leap.soledad.common import USER_DB_PREFIX diff --git a/server/src/leap/soledad/server/sync.py b/server/src/leap/soledad/server/sync.py index 96f65912..3f5c4aba 100644 --- a/server/src/leap/soledad/server/sync.py +++ b/server/src/leap/soledad/server/sync.py @@ -17,10 +17,10 @@ """ Server side synchronization infrastructure. """ -from u1db import sync, Document -from u1db.remote import http_app -from leap.soledad.server.state import ServerSyncState +from leap.soledad.common.l2db import sync, Document +from leap.soledad.common.l2db.remote import http_app from leap.soledad.server.caching import get_cache_for +from leap.soledad.server.state import ServerSyncState MAX_REQUEST_SIZE = 200 # in Mb -- cgit v1.2.3 From 8a3bbc6c81f10d8e00fcdd779784f327425f1942 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 5 Jul 2016 08:37:45 +0200 Subject: [refactor] remove u1db dep from support code --- client/pkg/generate_wheels.sh | 2 +- client/pkg/pip_install_requirements.sh | 2 +- client/pkg/requirements-latest.pip | 1 - .../examples/benchmarks/measure_index_times.py | 4 ++-- .../benchmarks/measure_index_times_custom_docid.py | 4 ++-- .../src/leap/soledad/client/examples/use_adbapi.py | 4 ++-- client/src/leap/soledad/client/sqlcipher.py | 2 +- common/pkg/generate_wheels.sh | 2 +- common/pkg/pip_install_requirements.sh | 2 +- common/pkg/requirements-latest.pip | 1 - scripts/docker/files/bin/setup-test-env.py | 19 ++++++++++--------- .../profiling/backends_cpu_usage/test_u1db_sync.py | 21 ++++++++------------- server/pkg/generate_wheels.sh | 2 +- server/pkg/pip_install_requirements.sh | 2 +- server/pkg/requirements-latest.pip | 1 - 15 files changed, 31 insertions(+), 38 deletions(-) diff --git a/client/pkg/generate_wheels.sh b/client/pkg/generate_wheels.sh index e29c327e..496f8e01 100755 --- a/client/pkg/generate_wheels.sh +++ b/client/pkg/generate_wheels.sh @@ -7,7 +7,7 @@ if [ "$WHEELHOUSE" = "" ]; then fi pip wheel --wheel-dir $WHEELHOUSE pip -pip wheel --wheel-dir $WHEELHOUSE --allow-external u1db --allow-unverified u1db --allow-external dirspec --allow-unverified dirspec -r pkg/requirements.pip +pip wheel --wheel-dir $WHEELHOUSE --allow-external dirspec --allow-unverified dirspec -r pkg/requirements.pip if [ -f pkg/requirements-testing.pip ]; then pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements-testing.pip fi diff --git a/client/pkg/pip_install_requirements.sh b/client/pkg/pip_install_requirements.sh index 1f5ac5f6..b97c826f 100755 --- a/client/pkg/pip_install_requirements.sh +++ b/client/pkg/pip_install_requirements.sh @@ -4,7 +4,7 @@ # Use at your own risk. # See $usage for help -insecure_packages="u1db dirspec" +insecure_packages="dirspec" leap_wheelhouse=https://lizard.leap.se/wheels show_help() { diff --git a/client/pkg/requirements-latest.pip b/client/pkg/requirements-latest.pip index a629aa57..fa483db7 100644 --- a/client/pkg/requirements-latest.pip +++ b/client/pkg/requirements-latest.pip @@ -1,6 +1,5 @@ --index-url https://pypi.python.org/simple/ ---allow-external u1db --allow-unverified u1db --allow-external dirspec --allow-unverified dirspec -e 'git+https://github.com/pixelated-project/leap_pycommon.git@develop#egg=leap.common' diff --git a/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py index 08775580..4fc91d9d 100644 --- a/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py +++ b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times.py @@ -24,9 +24,9 @@ import hashlib import os import sys -import u1db from twisted.internet import defer, reactor +from leap.soledad.common import l2db from leap.soledad.client import adbapi from leap.soledad.client.sqlcipher import SQLCipherOptions @@ -135,7 +135,7 @@ def countDocs(_): def printResult(r, **kwargs): if kwargs: debug(*kwargs.values()) - elif isinstance(r, u1db.Document): + elif isinstance(r, l2db.Document): debug(r.doc_id, r.content['number']) else: len_results = len(r[1]) diff --git a/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py index 9deba136..38ea18a3 100644 --- a/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py +++ b/client/src/leap/soledad/client/examples/benchmarks/measure_index_times_custom_docid.py @@ -24,11 +24,11 @@ import hashlib import os import sys -import u1db from twisted.internet import defer, reactor from leap.soledad.client import adbapi from leap.soledad.client.sqlcipher import SQLCipherOptions +from leap.soledad.common import l2db folder = os.environ.get("TMPDIR", "tmp") @@ -135,7 +135,7 @@ def countDocs(_): def printResult(r, **kwargs): if kwargs: debug(*kwargs.values()) - elif isinstance(r, u1db.Document): + elif isinstance(r, l2db.Document): debug(r.doc_id, r.content['number']) else: len_results = len(r[1]) diff --git a/client/src/leap/soledad/client/examples/use_adbapi.py b/client/src/leap/soledad/client/examples/use_adbapi.py index d7bd21f2..a2683836 100644 --- a/client/src/leap/soledad/client/examples/use_adbapi.py +++ b/client/src/leap/soledad/client/examples/use_adbapi.py @@ -21,11 +21,11 @@ from __future__ import print_function import datetime import os -import u1db from twisted.internet import defer, reactor from leap.soledad.client import adbapi from leap.soledad.client.sqlcipher import SQLCipherOptions +from leap.soledad.common import l2db folder = os.environ.get("TMPDIR", "tmp") @@ -68,7 +68,7 @@ def countDocs(_): def printResult(r): - if isinstance(r, u1db.Document): + if isinstance(r, l2db.Document): debug(r.doc_id, r.content['number']) else: len_results = len(r[1]) diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index bf2a50f1..f36c0b6a 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -594,7 +594,7 @@ class U1DBSQLiteBackend(sqlite_backend.SQLitePartialExpandDatabase): self._db_handle = conn self._real_replica_uid = None self._ensure_schema() - self._factory = u1db.Document + self._factory = l2db.Document class SoledadSQLCipherWrapper(SQLCipherDatabase): diff --git a/common/pkg/generate_wheels.sh b/common/pkg/generate_wheels.sh index e29c327e..496f8e01 100755 --- a/common/pkg/generate_wheels.sh +++ b/common/pkg/generate_wheels.sh @@ -7,7 +7,7 @@ if [ "$WHEELHOUSE" = "" ]; then fi pip wheel --wheel-dir $WHEELHOUSE pip -pip wheel --wheel-dir $WHEELHOUSE --allow-external u1db --allow-unverified u1db --allow-external dirspec --allow-unverified dirspec -r pkg/requirements.pip +pip wheel --wheel-dir $WHEELHOUSE --allow-external dirspec --allow-unverified dirspec -r pkg/requirements.pip if [ -f pkg/requirements-testing.pip ]; then pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements-testing.pip fi diff --git a/common/pkg/pip_install_requirements.sh b/common/pkg/pip_install_requirements.sh index 1f5ac5f6..b97c826f 100755 --- a/common/pkg/pip_install_requirements.sh +++ b/common/pkg/pip_install_requirements.sh @@ -4,7 +4,7 @@ # Use at your own risk. # See $usage for help -insecure_packages="u1db dirspec" +insecure_packages="dirspec" leap_wheelhouse=https://lizard.leap.se/wheels show_help() { diff --git a/common/pkg/requirements-latest.pip b/common/pkg/requirements-latest.pip index 9de17382..9b579503 100644 --- a/common/pkg/requirements-latest.pip +++ b/common/pkg/requirements-latest.pip @@ -1,6 +1,5 @@ --index-url https://pypi.python.org/simple/ ---allow-external u1db --allow-unverified u1db --allow-external dirspec --allow-unverified dirspec -e 'git+https://github.com/pixelated-project/leap_pycommon.git@develop#egg=leap.common' -e . diff --git a/scripts/docker/files/bin/setup-test-env.py b/scripts/docker/files/bin/setup-test-env.py index 0569b65d..0f3ea6f4 100755 --- a/scripts/docker/files/bin/setup-test-env.py +++ b/scripts/docker/files/bin/setup-test-env.py @@ -31,7 +31,8 @@ from couchdb.http import PreconditionFailed from couchdb.http import ResourceConflict from couchdb.http import ResourceNotFound from hashlib import sha512 -from u1db.errors import DatabaseDoesNotExist + +from leap.soledad.common.l2db.errors import DatabaseDoesNotExist # @@ -65,15 +66,15 @@ def pidfile_is_running(pidfile): return False -def status_from_pidfile(args, default_basedir): +def status_from_pidfile(args, default_basedir, name): basedir = _get_basedir(args, default_basedir) pidfile = os.path.join(basedir, args.pidfile) try: pid = get_pid(pidfile) psutil.Process(pid) - print "[+] running - pid: %d" % pid + print "[+] %s is running with pid %d" % (name, pid) except (IOError, psutil.NoSuchProcess): - print "[-] stopped" + print "[-] %s stopped" % name def kill_all_executables(args): @@ -163,7 +164,7 @@ def couch_server_start(args): except: time.sleep(0.1) - print '[+] running - pid: %d' % pid + print '[+] couch is running with pid: %d' % pid def couch_server_stop(args): @@ -177,11 +178,11 @@ def couch_server_stop(args): args.executable, '-p %s' % pidfile, # set the background PID FILE '-k']) # kill the background process, will respawn if needed - print '[-] stopped - pid: %d ' % pid + print '[-] stopped couch server with pid %d ' % pid def couch_status_from_pidfile(args): - status_from_pidfile(args, COUCH_BASEDIR) + status_from_pidfile(args, COUCH_BASEDIR, 'couch') # @@ -264,7 +265,7 @@ def soledad_server_start(args): call([args.executable] + params) pid = get_pid(pidfile) - print '[+] running - pid: %d' % pid + print '[+] soledad-server is running with pid %d' % pid def soledad_server_stop(args): @@ -279,7 +280,7 @@ def soledad_server_stop(args): def soledad_server_status_from_pidfile(args): - status_from_pidfile(args, SOLEDAD_SERVER_BASEDIR) + status_from_pidfile(args, SOLEDAD_SERVER_BASEDIR, 'soledad-server') # couch helpers diff --git a/scripts/profiling/backends_cpu_usage/test_u1db_sync.py b/scripts/profiling/backends_cpu_usage/test_u1db_sync.py index 26ef8f9f..5ae68c81 100755 --- a/scripts/profiling/backends_cpu_usage/test_u1db_sync.py +++ b/scripts/profiling/backends_cpu_usage/test_u1db_sync.py @@ -1,18 +1,16 @@ #!/usr/bin/python -import u1db import tempfile import logging import shutil import os -import argparse import time import binascii -import random - +from leap.soledad.common import l2db from leap.soledad.client.sqlcipher import open as sqlcipher_open + from log_cpu_usage import LogCpuUsage from u1dblite import open as u1dblite_open from u1dbcipher import open as u1dbcipher_open @@ -24,10 +22,10 @@ BIGGEST_DOC_SIZE = 100 * 1024 # 100 KB def get_data(size): - return binascii.hexlify(os.urandom(size/2)) + return binascii.hexlify(os.urandom(size / 2)) -def run_test(testname, open_fun, tempdir, docs, *args): +def run_test(testname, open_fun, tempdir, docs, *args): logger.info('Starting test \"%s\".' % testname) # instantiate dbs @@ -36,8 +34,7 @@ def run_test(testname, open_fun, tempdir, docs, *args): # get sync target and synchsonizer target = db2.get_sync_target() - synchronizer = u1db.sync.Synchronizer(db1, target) - + synchronizer = l2db.sync.Synchronizer(db1, target) # generate lots of small documents logger.info('Creating %d documents in source db...' % DOCS_TO_SYNC) @@ -80,30 +77,28 @@ def run_test(testname, open_fun, tempdir, docs, *args): if __name__ == '__main__': - + # configure logger logger = logging.getLogger(__name__) LOG_FORMAT = '%(asctime)s %(message)s' logging.basicConfig(format=LOG_FORMAT, level=logging.INFO) - # get a temporary dir tempdir = tempfile.mkdtemp() logger.info('Using temporary directory %s' % tempdir) - # create a lot of documents with random sizes docs = [] for i in xrange(DOCS_TO_SYNC): docs.append({ 'index': i, - #'data': get_data( + # 'data': get_data( # random.randrange( # SMALLEST_DOC_SIZE, BIGGEST_DOC_SIZE)) }) # run tests - run_test('sqlite', u1db.open, tempdir, docs, True) + run_test('sqlite', l2db.open, tempdir, docs, True) run_test('sqlcipher', sqlcipher_open, tempdir, docs, '123456', True) run_test('u1dblite', u1dblite_open, tempdir, docs) run_test('u1dbcipher', u1dbcipher_open, tempdir, docs, '123456', True) diff --git a/server/pkg/generate_wheels.sh b/server/pkg/generate_wheels.sh index e29c327e..496f8e01 100755 --- a/server/pkg/generate_wheels.sh +++ b/server/pkg/generate_wheels.sh @@ -7,7 +7,7 @@ if [ "$WHEELHOUSE" = "" ]; then fi pip wheel --wheel-dir $WHEELHOUSE pip -pip wheel --wheel-dir $WHEELHOUSE --allow-external u1db --allow-unverified u1db --allow-external dirspec --allow-unverified dirspec -r pkg/requirements.pip +pip wheel --wheel-dir $WHEELHOUSE --allow-external dirspec --allow-unverified dirspec -r pkg/requirements.pip if [ -f pkg/requirements-testing.pip ]; then pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements-testing.pip fi diff --git a/server/pkg/pip_install_requirements.sh b/server/pkg/pip_install_requirements.sh index 1f5ac5f6..b97c826f 100755 --- a/server/pkg/pip_install_requirements.sh +++ b/server/pkg/pip_install_requirements.sh @@ -4,7 +4,7 @@ # Use at your own risk. # See $usage for help -insecure_packages="u1db dirspec" +insecure_packages="dirspec" leap_wheelhouse=https://lizard.leap.se/wheels show_help() { diff --git a/server/pkg/requirements-latest.pip b/server/pkg/requirements-latest.pip index a629aa57..fa483db7 100644 --- a/server/pkg/requirements-latest.pip +++ b/server/pkg/requirements-latest.pip @@ -1,6 +1,5 @@ --index-url https://pypi.python.org/simple/ ---allow-external u1db --allow-unverified u1db --allow-external dirspec --allow-unverified dirspec -e 'git+https://github.com/pixelated-project/leap_pycommon.git@develop#egg=leap.common' -- cgit v1.2.3 From b3fb215860a8e50e4a6c551fef78628acdbf25c7 Mon Sep 17 00:00:00 2001 From: drebs Date: Tue, 5 Jul 2016 07:45:06 +0200 Subject: [bug] use default sqlcipher timeout --- client/src/leap/soledad/client/adbapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/leap/soledad/client/adbapi.py b/client/src/leap/soledad/client/adbapi.py index cfd7675c..328b4762 100644 --- a/client/src/leap/soledad/client/adbapi.py +++ b/client/src/leap/soledad/client/adbapi.py @@ -49,7 +49,7 @@ if DEBUG_SQL: How long the SQLCipher connection should wait for the lock to go away until raising an exception. """ -SQLCIPHER_CONNECTION_TIMEOUT = 10 +SQLCIPHER_CONNECTION_TIMEOUT = 5 """ How many times a SQLCipher query should be retried in case of timeout. -- cgit v1.2.3 From 26f87181f8a8fc7fef58ddd1e52cb5f0edd641bb Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 7 Jul 2016 11:44:01 +0200 Subject: [test] toxify tests - move tests to root directory - split tests in different subdirectories - setup a small package with common test dependencies in /testing/test_soledad - add tox.ini that will: - install the test_soledad package and other test dependencies - install soledad common, client, server from the repository - run tests contianed in /testing/tests directory using pytest This commit also removes all oauth code from tests, as we have removed the u1db dependency (by importing it into the repo and naming it l2db) and don't neet oauth at all right now. --- .../leap/soledad/common/tests/couchdb.ini.template | 22 - .../leap/soledad/common/tests/fixture_soledad.conf | 11 - .../leap/soledad/common/tests/hacker_crackdown.txt | 13005 ------------------- .../src/leap/soledad/common/tests/server_state.py | 81 - common/src/leap/soledad/common/tests/test_async.py | 142 - common/src/leap/soledad/common/tests/test_couch.py | 1442 -- .../tests/test_couch_operations_atomicity.py | 371 - .../src/leap/soledad/common/tests/test_crypto.py | 226 - .../leap/soledad/common/tests/test_encdecpool.py | 315 - common/src/leap/soledad/common/tests/test_http.py | 60 - .../leap/soledad/common/tests/test_http_client.py | 117 - common/src/leap/soledad/common/tests/test_https.py | 137 - .../src/leap/soledad/common/tests/test_server.py | 537 - .../src/leap/soledad/common/tests/test_soledad.py | 356 - .../leap/soledad/common/tests/test_soledad_app.py | 54 - .../leap/soledad/common/tests/test_soledad_doc.py | 46 - .../leap/soledad/common/tests/test_sqlcipher.py | 721 - .../soledad/common/tests/test_sqlcipher_sync.py | 743 -- common/src/leap/soledad/common/tests/test_sync.py | 216 - .../soledad/common/tests/test_sync_deferred.py | 196 - .../leap/soledad/common/tests/test_sync_mutex.py | 135 - .../leap/soledad/common/tests/test_sync_target.py | 956 -- .../leap/soledad/common/tests/u1db_tests/README | 25 - .../soledad/common/tests/u1db_tests/__init__.py | 461 - .../common/tests/u1db_tests/test_backends.py | 1915 --- .../common/tests/u1db_tests/test_document.py | 153 - .../common/tests/u1db_tests/test_http_client.py | 360 - .../common/tests/u1db_tests/test_http_database.py | 253 - .../soledad/common/tests/u1db_tests/test_https.py | 114 - .../soledad/common/tests/u1db_tests/test_open.py | 72 - .../common/tests/u1db_tests/testing-certs/Makefile | 35 - .../tests/u1db_tests/testing-certs/cacert.pem | 58 - .../tests/u1db_tests/testing-certs/testing.cert | 61 - .../tests/u1db_tests/testing-certs/testing.key | 16 - common/src/leap/soledad/common/tests/util.py | 419 - testing/setup.py | 9 + testing/test_soledad/__init__.py | 5 + testing/test_soledad/fixture_soledad.conf | 11 + testing/test_soledad/u1db_tests/README | 23 + testing/test_soledad/u1db_tests/__init__.py | 415 + testing/test_soledad/u1db_tests/test_backends.py | 1888 +++ testing/test_soledad/u1db_tests/test_document.py | 153 + .../test_soledad/u1db_tests/test_http_client.py | 304 + .../test_soledad/u1db_tests/test_http_database.py | 233 + testing/test_soledad/u1db_tests/test_https.py | 105 + testing/test_soledad/u1db_tests/test_open.py | 72 + .../test_soledad/u1db_tests/testing-certs/Makefile | 35 + .../u1db_tests/testing-certs/cacert.pem | 58 + .../u1db_tests/testing-certs/testing.cert | 61 + .../u1db_tests/testing-certs/testing.key | 16 + testing/test_soledad/util.py | 430 + testing/tests/client/__init__.py | 0 testing/tests/client/hacker_crackdown.txt | 13005 +++++++++++++++++++ testing/tests/client/test_app.py | 44 + testing/tests/client/test_async.py | 142 + testing/tests/client/test_aux_methods.py | 147 + testing/tests/client/test_crypto.py | 226 + testing/tests/client/test_doc.py | 46 + testing/tests/client/test_http.py | 60 + testing/tests/client/test_http_client.py | 108 + testing/tests/client/test_https.py | 137 + testing/tests/client/test_shared_db.py | 50 + testing/tests/client/test_signals.py | 165 + testing/tests/client/test_soledad_doc.py | 46 + testing/tests/client/test_sqlcipher.py | 705 + testing/tests/client/test_sqlcipher_sync.py | 730 ++ testing/tests/couch/__init__.py | 0 testing/tests/couch/couchdb.ini.template | 22 + testing/tests/couch/test_couch.py | 1442 ++ .../tests/couch/test_couch_operations_atomicity.py | 371 + testing/tests/server/__init__.py | 0 testing/tests/server/test_server.py | 536 + testing/tests/sync/__init__.py | 0 testing/tests/sync/test_encdecpool.py | 315 + testing/tests/sync/test_sync.py | 216 + testing/tests/sync/test_sync_deferred.py | 196 + testing/tests/sync/test_sync_mutex.py | 135 + testing/tests/sync/test_sync_target.py | 968 ++ testing/tox.ini | 21 + 79 files changed, 23651 insertions(+), 23831 deletions(-) delete mode 100644 common/src/leap/soledad/common/tests/couchdb.ini.template delete mode 100644 common/src/leap/soledad/common/tests/fixture_soledad.conf delete mode 100644 common/src/leap/soledad/common/tests/hacker_crackdown.txt delete mode 100644 common/src/leap/soledad/common/tests/server_state.py delete mode 100644 common/src/leap/soledad/common/tests/test_async.py delete mode 100644 common/src/leap/soledad/common/tests/test_couch.py delete mode 100644 common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py delete mode 100644 common/src/leap/soledad/common/tests/test_crypto.py delete mode 100644 common/src/leap/soledad/common/tests/test_encdecpool.py delete mode 100644 common/src/leap/soledad/common/tests/test_http.py delete mode 100644 common/src/leap/soledad/common/tests/test_http_client.py delete mode 100644 common/src/leap/soledad/common/tests/test_https.py delete mode 100644 common/src/leap/soledad/common/tests/test_server.py delete mode 100644 common/src/leap/soledad/common/tests/test_soledad.py delete mode 100644 common/src/leap/soledad/common/tests/test_soledad_app.py delete mode 100644 common/src/leap/soledad/common/tests/test_soledad_doc.py delete mode 100644 common/src/leap/soledad/common/tests/test_sqlcipher.py delete mode 100644 common/src/leap/soledad/common/tests/test_sqlcipher_sync.py delete mode 100644 common/src/leap/soledad/common/tests/test_sync.py delete mode 100644 common/src/leap/soledad/common/tests/test_sync_deferred.py delete mode 100644 common/src/leap/soledad/common/tests/test_sync_mutex.py delete mode 100644 common/src/leap/soledad/common/tests/test_sync_target.py delete mode 100644 common/src/leap/soledad/common/tests/u1db_tests/README delete mode 100644 common/src/leap/soledad/common/tests/u1db_tests/__init__.py delete mode 100644 common/src/leap/soledad/common/tests/u1db_tests/test_backends.py delete mode 100644 common/src/leap/soledad/common/tests/u1db_tests/test_document.py delete mode 100644 common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py delete mode 100644 common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py delete mode 100644 common/src/leap/soledad/common/tests/u1db_tests/test_https.py delete mode 100644 common/src/leap/soledad/common/tests/u1db_tests/test_open.py delete mode 100644 common/src/leap/soledad/common/tests/u1db_tests/testing-certs/Makefile delete mode 100644 common/src/leap/soledad/common/tests/u1db_tests/testing-certs/cacert.pem delete mode 100644 common/src/leap/soledad/common/tests/u1db_tests/testing-certs/testing.cert delete mode 100644 common/src/leap/soledad/common/tests/u1db_tests/testing-certs/testing.key delete mode 100644 common/src/leap/soledad/common/tests/util.py create mode 100644 testing/setup.py create mode 100644 testing/test_soledad/__init__.py create mode 100644 testing/test_soledad/fixture_soledad.conf create mode 100644 testing/test_soledad/u1db_tests/README create mode 100644 testing/test_soledad/u1db_tests/__init__.py create mode 100644 testing/test_soledad/u1db_tests/test_backends.py create mode 100644 testing/test_soledad/u1db_tests/test_document.py create mode 100644 testing/test_soledad/u1db_tests/test_http_client.py create mode 100644 testing/test_soledad/u1db_tests/test_http_database.py create mode 100644 testing/test_soledad/u1db_tests/test_https.py create mode 100644 testing/test_soledad/u1db_tests/test_open.py create mode 100644 testing/test_soledad/u1db_tests/testing-certs/Makefile create mode 100644 testing/test_soledad/u1db_tests/testing-certs/cacert.pem create mode 100644 testing/test_soledad/u1db_tests/testing-certs/testing.cert create mode 100644 testing/test_soledad/u1db_tests/testing-certs/testing.key create mode 100644 testing/test_soledad/util.py create mode 100644 testing/tests/client/__init__.py create mode 100644 testing/tests/client/hacker_crackdown.txt create mode 100644 testing/tests/client/test_app.py create mode 100644 testing/tests/client/test_async.py create mode 100644 testing/tests/client/test_aux_methods.py create mode 100644 testing/tests/client/test_crypto.py create mode 100644 testing/tests/client/test_doc.py create mode 100644 testing/tests/client/test_http.py create mode 100644 testing/tests/client/test_http_client.py create mode 100644 testing/tests/client/test_https.py create mode 100644 testing/tests/client/test_shared_db.py create mode 100644 testing/tests/client/test_signals.py create mode 100644 testing/tests/client/test_soledad_doc.py create mode 100644 testing/tests/client/test_sqlcipher.py create mode 100644 testing/tests/client/test_sqlcipher_sync.py create mode 100644 testing/tests/couch/__init__.py create mode 100644 testing/tests/couch/couchdb.ini.template create mode 100644 testing/tests/couch/test_couch.py create mode 100644 testing/tests/couch/test_couch_operations_atomicity.py create mode 100644 testing/tests/server/__init__.py create mode 100644 testing/tests/server/test_server.py create mode 100644 testing/tests/sync/__init__.py create mode 100644 testing/tests/sync/test_encdecpool.py create mode 100644 testing/tests/sync/test_sync.py create mode 100644 testing/tests/sync/test_sync_deferred.py create mode 100644 testing/tests/sync/test_sync_mutex.py create mode 100644 testing/tests/sync/test_sync_target.py create mode 100644 testing/tox.ini diff --git a/common/src/leap/soledad/common/tests/couchdb.ini.template b/common/src/leap/soledad/common/tests/couchdb.ini.template deleted file mode 100644 index 174d9d86..00000000 --- a/common/src/leap/soledad/common/tests/couchdb.ini.template +++ /dev/null @@ -1,22 +0,0 @@ -; etc/couchdb/default.ini.tpl. Generated from default.ini.tpl.in by configure. - -; Upgrading CouchDB will overwrite this file. - -[couchdb] -database_dir = %(tempdir)s/lib -view_index_dir = %(tempdir)s/lib -max_document_size = 4294967296 ; 4 GB -os_process_timeout = 120000 ; 120 seconds. for view and external servers. -max_dbs_open = 100 -delayed_commits = true ; set this to false to ensure an fsync before 201 Created is returned -uri_file = %(tempdir)s/lib/couch.uri -file_compression = snappy - -[log] -file = %(tempdir)s/log/couch.log -level = info -include_sasl = true - -[httpd] -port = 0 -bind_address = 127.0.0.1 diff --git a/common/src/leap/soledad/common/tests/fixture_soledad.conf b/common/src/leap/soledad/common/tests/fixture_soledad.conf deleted file mode 100644 index 8d8161c3..00000000 --- a/common/src/leap/soledad/common/tests/fixture_soledad.conf +++ /dev/null @@ -1,11 +0,0 @@ -[soledad-server] -couch_url = http://soledad:passwd@localhost:5984 -create_cmd = sudo -u soledad-admin /usr/bin/create-user-db -admin_netrc = /etc/couchdb/couchdb-soledad-admin.netrc -batching = 0 - -[database-security] -members = user1, user2 -members_roles = role1, role2 -admins = user3, user4 -admins_roles = role3, role3 diff --git a/common/src/leap/soledad/common/tests/hacker_crackdown.txt b/common/src/leap/soledad/common/tests/hacker_crackdown.txt deleted file mode 100644 index a01eb509..00000000 --- a/common/src/leap/soledad/common/tests/hacker_crackdown.txt +++ /dev/null @@ -1,13005 +0,0 @@ -The Project Gutenberg EBook of Hacker Crackdown, by Bruce Sterling - -This eBook is for the use of anyone anywhere at no cost and with -almost no restrictions whatsoever. You may copy it, give it away or -re-use it under the terms of the Project Gutenberg License included -with this eBook or online at www.gutenberg.org - -** This is a COPYRIGHTED Project Gutenberg eBook, Details Below ** -** Please follow the copyright guidelines in this file. ** - -Title: Hacker Crackdown - Law and Disorder on the Electronic Frontier - -Author: Bruce Sterling - -Posting Date: February 9, 2012 [EBook #101] -Release Date: January, 1994 - -Language: English - - -*** START OF THIS PROJECT GUTENBERG EBOOK HACKER CRACKDOWN *** - - - - - - - - - - - - - -THE HACKER CRACKDOWN - -Law and Disorder on the Electronic Frontier - -by Bruce Sterling - - - - -CONTENTS - - -Preface to the Electronic Release of The Hacker Crackdown - -Chronology of the Hacker Crackdown - - -Introduction - - -Part 1: CRASHING THE SYSTEM -A Brief History of Telephony -Bell's Golden Vaporware -Universal Service -Wild Boys and Wire Women -The Electronic Communities -The Ungentle Giant -The Breakup -In Defense of the System -The Crash Post-Mortem -Landslides in Cyberspace - - -Part 2: THE DIGITAL UNDERGROUND -Steal This Phone -Phreaking and Hacking -The View From Under the Floorboards -Boards: Core of the Underground -Phile Phun -The Rake's Progress -Strongholds of the Elite -Sting Boards -Hot Potatoes -War on the Legion -Terminus -Phile 9-1-1 -War Games -Real Cyberpunk - - -Part 3: LAW AND ORDER -Crooked Boards -The World's Biggest Hacker Bust -Teach Them a Lesson -The U.S. Secret Service -The Secret Service Battles the Boodlers -A Walk Downtown -FCIC: The Cutting-Edge Mess -Cyberspace Rangers -FLETC: Training the Hacker-Trackers - - -Part 4: THE CIVIL LIBERTARIANS -NuPrometheus + FBI = Grateful Dead -Whole Earth + Computer Revolution = WELL -Phiber Runs Underground and Acid Spikes the Well -The Trial of Knight Lightning -Shadowhawk Plummets to Earth -Kyrie in the Confessional -$79,499 -A Scholar Investigates -Computers, Freedom, and Privacy - - -Electronic Afterword to The Hacker Crackdown, Halloween 1993 - - - - -THE HACKER CRACKDOWN - -Law and Disorder on the Electronic Frontier - -by Bruce Sterling - - - - - -Preface to the Electronic Release of The Hacker Crackdown - - -January 1, 1994--Austin, Texas - - -Hi, I'm Bruce Sterling, the author of this electronic book. - -Out in the traditional world of print, The Hacker Crackdown -is ISBN 0-553-08058-X, and is formally catalogued by -the Library of Congress as "1. Computer crimes--United States. -2. Telephone--United States--Corrupt practices. -3. Programming (Electronic computers)--United States--Corrupt practices." - -`Corrupt practices,' I always get a kick out of that description. -Librarians are very ingenious people. - -The paperback is ISBN 0-553-56370-X. If you go -and buy a print version of The Hacker Crackdown, -an action I encourage heartily, you may notice that -in the front of the book, beneath the copyright notice-- -"Copyright (C) 1992 by Bruce Sterling"-- -it has this little block of printed legal -boilerplate from the publisher. It says, and I quote: - - "No part of this book may be reproduced or transmitted in any form -or by any means, electronic or mechanical, including photocopying, -recording, or by any information storage and retrieval system, -without permission in writing from the publisher. -For information address: Bantam Books." - -This is a pretty good disclaimer, as such disclaimers go. -I collect intellectual-property disclaimers, and I've seen dozens of them, -and this one is at least pretty straightforward. In this narrow -and particular case, however, it isn't quite accurate. -Bantam Books puts that disclaimer on every book they publish, -but Bantam Books does not, in fact, own the electronic rights to this book. -I do, because of certain extensive contract maneuverings my agent and I -went through before this book was written. I want to give those electronic -publishing rights away through certain not-for-profit channels, -and I've convinced Bantam that this is a good idea. - -Since Bantam has seen fit to peacably agree to this scheme of mine, -Bantam Books is not going to fuss about this. Provided you don't try -to sell the book, they are not going to bother you for what you do with -the electronic copy of this book. If you want to check this out personally, -you can ask them; they're at 1540 Broadway NY NY 10036. However, if you were -so foolish as to print this book and start retailing it for money in violation -of my copyright and the commercial interests of Bantam Books, then Bantam, -a part of the gigantic Bertelsmann multinational publishing combine, -would roust some of their heavy-duty attorneys out of hibernation -and crush you like a bug. This is only to be expected. -I didn't write this book so that you could make money out of it. -If anybody is gonna make money out of this book, -it's gonna be me and my publisher. - -My publisher deserves to make money out of this book. -Not only did the folks at Bantam Books commission me -to write the book, and pay me a hefty sum to do so, but -they bravely printed, in text, an electronic document the -reproduction of which was once alleged to be a federal felony. -Bantam Books and their numerous attorneys were very brave -and forthright about this book. Furthermore, my former editor -at Bantam Books, Betsy Mitchell, genuinely cared about this project, -and worked hard on it, and had a lot of wise things to say -about the manuscript. Betsy deserves genuine credit for this book, -credit that editors too rarely get. - -The critics were very kind to The Hacker Crackdown, -and commercially the book has done well. On the other hand, -I didn't write this book in order to squeeze every last nickel -and dime out of the mitts of impoverished sixteen-year-old -cyberpunk high-school-students. Teenagers don't have any money-- -(no, not even enough for the six-dollar Hacker Crackdown paperback, -with its attractive bright-red cover and useful index). -That's a major reason why teenagers sometimes succumb to the temptation -to do things they shouldn't, such as swiping my books out of libraries. -Kids: this one is all yours, all right? Go give the print version back. -*8-) - -Well-meaning, public-spirited civil libertarians don't have much money, -either. And it seems almost criminal to snatch cash out of the hands of -America's direly underpaid electronic law enforcement community. - -If you're a computer cop, a hacker, or an electronic civil -liberties activist, you are the target audience for this book. -I wrote this book because I wanted to help you, and help other people -understand you and your unique, uhm, problems. I wrote this book -to aid your activities, and to contribute to the public discussion -of important political issues. In giving the text away in this -fashion, I am directly contributing to the book's ultimate aim: -to help civilize cyberspace. - -Information WANTS to be free. And the information inside -this book longs for freedom with a peculiar intensity. -I genuinely believe that the natural habitat of this book -is inside an electronic network. That may not be the easiest -direct method to generate revenue for the book's author, -but that doesn't matter; this is where this book belongs -by its nature. I've written other books--plenty of other books-- -and I'll write more and I am writing more, but this one is special. -I am making The Hacker Crackdown available electronically -as widely as I can conveniently manage, and if you like the book, -and think it is useful, then I urge you to do the same with it. - -You can copy this electronic book. Copy the heck out of it, -be my guest, and give those copies to anybody who wants them. -The nascent world of cyberspace is full of sysadmins, teachers, -trainers, cybrarians, netgurus, and various species of cybernetic activist. -If you're one of those people, I know about you, and I know the hassle -you go through to try to help people learn about the electronic frontier. -I hope that possessing this book in electronic form will lessen your troubles. -Granted, this treatment of our electronic social spectrum is not the ultimate -in academic rigor. And politically, it has something to offend -and trouble almost everyone. But hey, I'm told it's readable, -and at least the price is right. - -You can upload the book onto bulletin board systems, or Internet nodes, -or electronic discussion groups. Go right ahead and do that, I am giving -you express permission right now. Enjoy yourself. - -You can put the book on disks and give the disks away, -as long as you don't take any money for it. - -But this book is not public domain. You can't copyright it in -your own name. I own the copyright. Attempts to pirate this book -and make money from selling it may involve you in a serious litigative snarl. -Believe me, for the pittance you might wring out of such an action, -it's really not worth it. This book don't "belong" to you. -In an odd but very genuine way, I feel it doesn't "belong" to me, either. -It's a book about the people of cyberspace, and distributing it in this way -is the best way I know to actually make this information available, -freely and easily, to all the people of cyberspace--including people -far outside the borders of the United States, who otherwise may never -have a chance to see any edition of the book, and who may perhaps learn -something useful from this strange story of distant, obscure, but portentous -events in so-called "American cyberspace." - -This electronic book is now literary freeware. It now belongs to the -emergent realm of alternative information economics. You have no right -to make this electronic book part of the conventional flow of commerce. -Let it be part of the flow of knowledge: there's a difference. -I've divided the book into four sections, so that it is less ungainly -for upload and download; if there's a section of particular relevance -to you and your colleagues, feel free to reproduce that one and skip the rest. - -[Project Gutenberg has reassembled the file, with Sterling's permission.] - -Just make more when you need them, and give them to whoever might want them. - -Now have fun. - -Bruce Sterling--bruces@well.sf.ca.us - - -THE HACKER CRACKDOWN - -Law and Disorder on the Electronic Frontier - -by Bruce Sterling - - - - - - - -CHRONOLOGY OF THE HACKER CRACKDOWN - - -1865 U.S. Secret Service (USSS) founded. - -1876 Alexander Graham Bell invents telephone. - -1878 First teenage males flung off phone system by enraged authorities. - -1939 "Futurian" science-fiction group raided by Secret Service. - -1971 Yippie phone phreaks start YIPL/TAP magazine. - -1972 RAMPARTS magazine seized in blue-box rip-off scandal. - -1978 Ward Christenson and Randy Suess create first personal - computer bulletin board system. - -1982 William Gibson coins term "cyberspace." - -1982 "414 Gang" raided. - -1983-1983 AT&T dismantled in divestiture. - -1984 Congress passes Comprehensive Crime Control Act giving USSS - jurisdiction over credit card fraud and computer fraud. - -1984 "Legion of Doom" formed. - -1984. 2600: THE HACKER QUARTERLY founded. - -1984. WHOLE EARTH SOFTWARE CATALOG published. - -1985. First police "sting" bulletin board systems established. - -1985. Whole Earth 'Lectronic Link computer conference (WELL) goes on-line. - -1986 Computer Fraud and Abuse Act passed. - -1986 Electronic Communications Privacy Act passed. - -1987 Chicago prosecutors form Computer Fraud and Abuse Task Force. - - -1988 - -July. Secret Service covertly videotapes "SummerCon" hacker convention. - -September. "Prophet" cracks BellSouth AIMSX computer network - and downloads E911 Document to his own computer and to Jolnet. - -September. AT&T Corporate Information Security informed of Prophet's action. - -October. Bellcore Security informed of Prophet's action. - - -1989 - -January. Prophet uploads E911 Document to Knight Lightning. - -February 25. Knight Lightning publishes E911 Document in PHRACK - electronic newsletter. - -May. Chicago Task Force raids and arrests "Kyrie." - -June. "NuPrometheus League" distributes Apple Computer proprietary software. - -June 13. Florida probation office crossed with phone-sex line - in switching-station stunt. - -July. "Fry Guy" raided by USSS and Chicago Computer Fraud - and Abuse Task Force. - -July. Secret Service raids "Prophet," "Leftist," and "Urvile" in Georgia. - - -1990 - -January 15. Martin Luther King Day Crash strikes AT&T long-distance - network nationwide. - -January 18-19. Chicago Task Force raids Knight Lightning in St. Louis. - -January 24. USSS and New York State Police raid "Phiber Optik," - "Acid Phreak," and "Scorpion" in New York City. - -February 1. USSS raids "Terminus" in Maryland. - -February 3. Chicago Task Force raids Richard Andrews' home. - -February 6. Chicago Task Force raids Richard Andrews' business. - -February 6. USSS arrests Terminus, Prophet, Leftist, and Urvile. - -February 9. Chicago Task Force arrests Knight Lightning. - -February 20. AT&T Security shuts down public-access - "attctc" computer in Dallas. - -February 21. Chicago Task Force raids Robert Izenberg in Austin. - -March 1. Chicago Task Force raids Steve Jackson Games, Inc., - "Mentor," and "Erik Bloodaxe" in Austin. - -May 7,8,9. - -USSS and Arizona Organized Crime and Racketeering Bureau conduct -"Operation Sundevil" raids in Cincinnatti, Detroit, Los Angeles, -Miami, Newark, Phoenix, Pittsburgh, Richmond, Tucson, San Diego, -San Jose, and San Francisco. - -May. FBI interviews John Perry Barlow re NuPrometheus case. - -June. Mitch Kapor and Barlow found Electronic Frontier Foundation; - Barlow publishes CRIME AND PUZZLEMENT manifesto. - -July 24-27. Trial of Knight Lightning. - -1991 - -February. CPSR Roundtable in Washington, D.C. - -March 25-28. Computers, Freedom and Privacy conference in San Francisco. - -May 1. Electronic Frontier Foundation, Steve Jackson, - and others file suit against members of Chicago Task Force. - -July 1-2. Switching station phone software crash affects - Washington, Los Angeles, Pittsburgh, San Francisco. - -September 17. AT&T phone crash affects New York City and three airports. - - - - -Introduction - -This is a book about cops, and wild teenage whiz-kids, and lawyers, -and hairy-eyed anarchists, and industrial technicians, and hippies, -and high-tech millionaires, and game hobbyists, and computer security -experts, and Secret Service agents, and grifters, and thieves. - -This book is about the electronic frontier of the 1990s. -It concerns activities that take place inside computers -and over telephone lines. - -A science fiction writer coined the useful term "cyberspace" in 1982, -but the territory in question, the electronic frontier, is about -a hundred and thirty years old. Cyberspace is the "place" where -a telephone conversation appears to occur. Not inside your actual phone, -the plastic device on your desk. Not inside the other person's phone, -in some other city. THE PLACE BETWEEN the phones. The indefinite -place OUT THERE, where the two of you, two human beings, -actually meet and communicate. - -Although it is not exactly "real," "cyberspace" is a genuine place. -Things happen there that have very genuine consequences. This "place" -is not "real," but it is serious, it is earnest. Tens of thousands -of people have dedicated their lives to it, to the public service -of public communication by wire and electronics. - -People have worked on this "frontier" for generations now. -Some people became rich and famous from their efforts there. -Some just played in it, as hobbyists. Others soberly pondered it, -and wrote about it, and regulated it, and negotiated over it in -international forums, and sued one another about it, in gigantic, -epic court battles that lasted for years. And almost since -the beginning, some people have committed crimes in this place. - -But in the past twenty years, this electrical "space," -which was once thin and dark and one-dimensional--little more -than a narrow speaking-tube, stretching from phone to phone-- -has flung itself open like a gigantic jack-in-the-box. -Light has flooded upon it, the eerie light of the glowing computer screen. -This dark electric netherworld has become a vast flowering electronic landscape. -Since the 1960s, the world of the telephone has cross-bred itself -with computers and television, and though there is still no substance -to cyberspace, nothing you can handle, it has a strange kind -of physicality now. It makes good sense today to talk of cyberspace -as a place all its own. - -Because people live in it now. Not just a few people, -not just a few technicians and eccentrics, but thousands -of people, quite normal people. And not just for a little while, -either, but for hours straight, over weeks, and months, -and years. Cyberspace today is a "Net," a "Matrix," -international in scope and growing swiftly and steadily. -It's growing in size, and wealth, and political importance. - -People are making entire careers in modern cyberspace. -Scientists and technicians, of course; they've been there -for twenty years now. But increasingly, cyberspace -is filling with journalists and doctors and lawyers -and artists and clerks. Civil servants make their -careers there now, "on-line" in vast government data-banks; -and so do spies, industrial, political, and just plain snoops; -and so do police, at least a few of them. And there are children -living there now. - -People have met there and been married there. -There are entire living communities in cyberspace today; -chattering, gossiping, planning, conferring and scheming, -leaving one another voice-mail and electronic mail, -giving one another big weightless chunks of valuable data, -both legitimate and illegitimate. They busily pass one another -computer software and the occasional festering computer virus. - -We do not really understand how to live in cyberspace yet. -We are feeling our way into it, blundering about. -That is not surprising. Our lives in the physical world, -the "real" world, are also far from perfect, despite a lot more practice. -Human lives, real lives, are imperfect by their nature, and there are -human beings in cyberspace. The way we live in cyberspace is -a funhouse mirror of the way we live in the real world. -We take both our advantages and our troubles with us. - -This book is about trouble in cyberspace. -Specifically, this book is about certain strange events in -the year 1990, an unprecedented and startling year for the -the growing world of computerized communications. - -In 1990 there came a nationwide crackdown on illicit -computer hackers, with arrests, criminal charges, -one dramatic show-trial, several guilty pleas, and -huge confiscations of data and equipment all over the USA. - -The Hacker Crackdown of 1990 was larger, better organized, -more deliberate, and more resolute than any previous effort -in the brave new world of computer crime. The U.S. Secret Service, -private telephone security, and state and local law enforcement groups -across the country all joined forces in a determined attempt to break -the back of America's electronic underground. It was a fascinating -effort, with very mixed results. - -The Hacker Crackdown had another unprecedented effect; -it spurred the creation, within "the computer community," -of the Electronic Frontier Foundation, a new and very odd -interest group, fiercely dedicated to the establishment -and preservation of electronic civil liberties. The crackdown, -remarkable in itself, has created a melee of debate over electronic crime, -punishment, freedom of the press, and issues of search and seizure. -Politics has entered cyberspace. Where people go, politics follow. - -This is the story of the people of cyberspace. - - - -PART ONE: Crashing the System - -On January 15, 1990, AT&T's long-distance telephone switching system crashed. - -This was a strange, dire, huge event. Sixty thousand people lost -their telephone service completely. During the nine long hours -of frantic effort that it took to restore service, some seventy million -telephone calls went uncompleted. - -Losses of service, known as "outages" in the telco trade, -are a known and accepted hazard of the telephone business. -Hurricanes hit, and phone cables get snapped by the thousands. -Earthquakes wrench through buried fiber-optic lines. -Switching stations catch fire and burn to the ground. -These things do happen. There are contingency plans for them, -and decades of experience in dealing with them. -But the Crash of January 15 was unprecedented. -It was unbelievably huge, and it occurred for -no apparent physical reason. - -The crash started on a Monday afternoon in a single -switching-station in Manhattan. But, unlike any merely -physical damage, it spread and spread. Station after -station across America collapsed in a chain reaction, -until fully half of AT&T's network had gone haywire -and the remaining half was hard-put to handle the overflow. - -Within nine hours, AT&T software engineers more or less -understood what had caused the crash. Replicating the -problem exactly, poring over software line by line, -took them a couple of weeks. But because it was hard -to understand technically, the full truth of the matter -and its implications were not widely and thoroughly aired -and explained. The root cause of the crash remained obscure, -surrounded by rumor and fear. - -The crash was a grave corporate embarrassment. -The "culprit" was a bug in AT&T's own software--not the -sort of admission the telecommunications giant wanted -to make, especially in the face of increasing competition. -Still, the truth WAS told, in the baffling technical terms -necessary to explain it. - -Somehow the explanation failed to persuade -American law enforcement officials and even telephone -corporate security personnel. These people were not -technical experts or software wizards, and they had their -own suspicions about the cause of this disaster. - -The police and telco security had important sources -of information denied to mere software engineers. -They had informants in the computer underground and -years of experience in dealing with high-tech rascality -that seemed to grow ever more sophisticated. -For years they had been expecting a direct and -savage attack against the American national telephone system. -And with the Crash of January 15--the first month of a -new, high-tech decade--their predictions, fears, -and suspicions seemed at last to have entered the real world. -A world where the telephone system had not merely crashed, -but, quite likely, BEEN crashed--by "hackers." - -The crash created a large dark cloud of suspicion -that would color certain people's assumptions and actions -for months. The fact that it took place in the realm of -software was suspicious on its face. The fact that it -occurred on Martin Luther King Day, still the most -politically touchy of American holidays, made it more -suspicious yet. - -The Crash of January 15 gave the Hacker Crackdown -its sense of edge and its sweaty urgency. It made people, -powerful people in positions of public authority, -willing to believe the worst. And, most fatally, -it helped to give investigators a willingness -to take extreme measures and the determination -to preserve almost total secrecy. - -An obscure software fault in an aging switching system -in New York was to lead to a chain reaction of legal -and constitutional trouble all across the country. - -# - -Like the crash in the telephone system, this chain reaction -was ready and waiting to happen. During the 1980s, -the American legal system was extensively patched -to deal with the novel issues of computer crime. -There was, for instance, the Electronic Communications -Privacy Act of 1986 (eloquently described as "a stinking mess" -by a prominent law enforcement official). And there was the -draconian Computer Fraud and Abuse Act of 1986, passed unanimously -by the United States Senate, which later would reveal -a large number of flaws. Extensive, well-meant efforts -had been made to keep the legal system up to date. -But in the day-to-day grind of the real world, -even the most elegant software tends to crumble -and suddenly reveal its hidden bugs. - -Like the advancing telephone system, the American legal system -was certainly not ruined by its temporary crash; but for those -caught under the weight of the collapsing system, life became -a series of blackouts and anomalies. - -In order to understand why these weird events occurred, -both in the world of technology and in the world of law, -it's not enough to understand the merely technical problems. -We will get to those; but first and foremost, we must try -to understand the telephone, and the business of telephones, -and the community of human beings that telephones have created. - -# - -Technologies have life cycles, like cities do, -like institutions do, like laws and governments do. - -The first stage of any technology is the Question -Mark, often known as the "Golden Vaporware" stage. -At this early point, the technology is only a phantom, -a mere gleam in the inventor's eye. One such inventor -was a speech teacher and electrical tinkerer named -Alexander Graham Bell. - -Bell's early inventions, while ingenious, failed to move the world. -In 1863, the teenage Bell and his brother Melville made an artificial -talking mechanism out of wood, rubber, gutta-percha, and tin. -This weird device had a rubber-covered "tongue" made of movable -wooden segments, with vibrating rubber "vocal cords," and -rubber "lips" and "cheeks." While Melville puffed a bellows -into a tin tube, imitating the lungs, young Alec Bell would -manipulate the "lips," "teeth," and "tongue," causing the thing -to emit high-pitched falsetto gibberish. - -Another would-be technical breakthrough was the Bell "phonautograph" -of 1874, actually made out of a human cadaver's ear. Clamped into place -on a tripod, this grisly gadget drew sound-wave images on smoked glass -through a thin straw glued to its vibrating earbones. - -By 1875, Bell had learned to produce audible sounds--ugly shrieks -and squawks--by using magnets, diaphragms, and electrical current. - -Most "Golden Vaporware" technologies go nowhere. - -But the second stage of technology is the Rising Star, -or, the "Goofy Prototype," stage. The telephone, Bell's -most ambitious gadget yet, reached this stage on March -10, 1876. On that great day, Alexander Graham Bell -became the first person to transmit intelligible human -speech electrically. As it happened, young Professor Bell, -industriously tinkering in his Boston lab, had spattered -his trousers with acid. His assistant, Mr. Watson, -heard his cry for help--over Bell's experimental -audio-telegraph. This was an event without precedent. - -Technologies in their "Goofy Prototype" stage rarely -work very well. They're experimental, and therefore -half- baked and rather frazzled. The prototype may -be attractive and novel, and it does look as if it ought -to be good for something-or-other. But nobody, including -the inventor, is quite sure what. Inventors, and speculators, -and pundits may have very firm ideas about its potential -use, but those ideas are often very wrong. - -The natural habitat of the Goofy Prototype is in trade shows -and in the popular press. Infant technologies need publicity -and investment money like a tottering calf need milk. -This was very true of Bell's machine. To raise research and -development money, Bell toured with his device as a stage attraction. - -Contemporary press reports of the stage debut of the telephone -showed pleased astonishment mixed with considerable dread. -Bell's stage telephone was a large wooden box with a crude -speaker-nozzle, the whole contraption about the size and shape -of an overgrown Brownie camera. Its buzzing steel soundplate, -pumped up by powerful electromagnets, was loud enough to fill -an auditorium. Bell's assistant Mr. Watson, who could manage -on the keyboards fairly well, kicked in by playing the organ -from distant rooms, and, later, distant cities. This feat was -considered marvellous, but very eerie indeed. - -Bell's original notion for the telephone, an idea promoted -for a couple of years, was that it would become a mass medium. -We might recognize Bell's idea today as something close to modern -"cable radio." Telephones at a central source would transmit music, -Sunday sermons, and important public speeches to a paying network -of wired-up subscribers. - -At the time, most people thought this notion made good sense. -In fact, Bell's idea was workable. In Hungary, this philosophy -of the telephone was successfully put into everyday practice. -In Budapest, for decades, from 1893 until after World War I, -there was a government-run information service called -"Telefon Hirmondo-." Hirmondo- was a centralized source -of news and entertainment and culture, including stock reports, -plays, concerts, and novels read aloud. At certain hours -of the day, the phone would ring, you would plug in -a loudspeaker for the use of the family, and Telefon -Hirmondo- would be on the air--or rather, on the phone. - -Hirmondo- is dead tech today, but Hirmondo- might be considered -a spiritual ancestor of the modern telephone-accessed computer -data services, such as CompuServe, GEnie or Prodigy. -The principle behind Hirmondo- is also not too far from computer -"bulletin- board systems" or BBS's, which arrived in the late 1970s, -spread rapidly across America, and will figure largely in this book. - -We are used to using telephones for individual person-to-person speech, -because we are used to the Bell system. But this was just one possibility -among many. Communication networks are very flexible and protean, -especially when their hardware becomes sufficiently advanced. -They can be put to all kinds of uses. And they have been-- -and they will be. - -Bell's telephone was bound for glory, but this was a combination -of political decisions, canny infighting in court, inspired industrial -leadership, receptive local conditions and outright good luck. -Much the same is true of communications systems today. - -As Bell and his backers struggled to install their newfangled system -in the real world of nineteenth-century New England, they had to fight -against skepticism and industrial rivalry. There was already a strong -electrical communications network present in America: the telegraph. -The head of the Western Union telegraph system dismissed Bell's prototype -as "an electrical toy" and refused to buy the rights to Bell's patent. -The telephone, it seemed, might be all right as a parlor entertainment-- -but not for serious business. - -Telegrams, unlike mere telephones, left a permanent physical record -of their messages. Telegrams, unlike telephones, could be answered -whenever the recipient had time and convenience. And the telegram -had a much longer distance-range than Bell's early telephone. -These factors made telegraphy seem a much more sound and businesslike -technology--at least to some. - -The telegraph system was huge, and well-entrenched. -In 1876, the United States had 214,000 miles of telegraph wire, -and 8500 telegraph offices. There were specialized telegraphs -for businesses and stock traders, government, police and fire departments. -And Bell's "toy" was best known as a stage-magic musical device. - -The third stage of technology is known as the "Cash Cow" stage. -In the "cash cow" stage, a technology finds its place in the world, -and matures, and becomes settled and productive. After a year or so, -Alexander Graham Bell and his capitalist backers concluded that -eerie music piped from nineteenth-century cyberspace was not the real -selling-point of his invention. Instead, the telephone was about speech-- -individual, personal speech, the human voice, human conversation and -human interaction. The telephone was not to be managed from any centralized -broadcast center. It was to be a personal, intimate technology. - -When you picked up a telephone, you were not absorbing -the cold output of a machine--you were speaking to another human being. -Once people realized this, their instinctive dread of the telephone -as an eerie, unnatural device, swiftly vanished. A "telephone call" -was not a "call" from a "telephone" itself, but a call from another -human being, someone you would generally know and recognize. -The real point was not what the machine could do for you (or to you), -but what you yourself, a person and citizen, could do THROUGH the machine. -This decision on the part of the young Bell Company was absolutely vital. - -The first telephone networks went up around Boston--mostly among -the technically curious and the well-to-do (much the same segment -of the American populace that, a hundred years later, would be -buying personal computers). Entrenched backers of the telegraph -continued to scoff. - -But in January 1878, a disaster made the telephone famous. -A train crashed in Tarriffville, Connecticut. Forward-looking -doctors in the nearby city of Hartford had had Bell's -"speaking telephone" installed. An alert local druggist -was able to telephone an entire community of local doctors, -who rushed to the site to give aid. The disaster, as disasters do, -aroused intense press coverage. The phone had proven its usefulness -in the real world. - -After Tarriffville, the telephone network spread like crabgrass. -By 1890 it was all over New England. By '93, out to Chicago. -By '97, into Minnesota, Nebraska and Texas. By 1904 it was -all over the continent. - -The telephone had become a mature technology. Professor Bell -(now generally known as "Dr. Bell" despite his lack of a formal degree) -became quite wealthy. He lost interest in the tedious day-to-day business -muddle of the booming telephone network, and gratefully returned -his attention to creatively hacking-around in his various laboratories, -which were now much larger, better-ventilated, and gratifyingly -better-equipped. Bell was never to have another great inventive success, -though his speculations and prototypes anticipated fiber-optic transmission, -manned flight, sonar, hydrofoil ships, tetrahedral construction, and -Montessori education. The "decibel," the standard scientific measure -of sound intensity, was named after Bell. - -Not all Bell's vaporware notions were inspired. He was fascinated -by human eugenics. He also spent many years developing a weird personal -system of astrophysics in which gravity did not exist. - -Bell was a definite eccentric. He was something of a hypochondriac, -and throughout his life he habitually stayed up until four A.M., -refusing to rise before noon. But Bell had accomplished a great feat; -he was an idol of millions and his influence, wealth, and great -personal charm, combined with his eccentricity, made him something -of a loose cannon on deck. Bell maintained a thriving scientific -salon in his winter mansion in Washington, D.C., which gave him -considerable backstage influence in governmental and scientific circles. -He was a major financial backer of the the magazines Science and -National Geographic, both still flourishing today as important organs -of the American scientific establishment. - -Bell's companion Thomas Watson, similarly wealthy and similarly odd, -became the ardent political disciple of a 19th-century science-fiction writer -and would-be social reformer, Edward Bellamy. Watson also trod the boards -briefly as a Shakespearian actor. - -There would never be another Alexander Graham Bell, -but in years to come there would be surprising numbers -of people like him. Bell was a prototype of the -high-tech entrepreneur. High-tech entrepreneurs will -play a very prominent role in this book: not merely as -technicians and businessmen, but as pioneers of the -technical frontier, who can carry the power and prestige -they derive from high-technology into the political and -social arena. - -Like later entrepreneurs, Bell was fierce in defense of -his own technological territory. As the telephone began to -flourish, Bell was soon involved in violent lawsuits in the -defense of his patents. Bell's Boston lawyers were -excellent, however, and Bell himself, as an elocution -teacher and gifted public speaker, was a devastatingly -effective legal witness. In the eighteen years of Bell's patents, -the Bell company was involved in six hundred separate lawsuits. -The legal records printed filled 149 volumes. The Bell Company -won every single suit. - -After Bell's exclusive patents expired, rival telephone -companies sprang up all over America. Bell's company, -American Bell Telephone, was soon in deep trouble. -In 1907, American Bell Telephone fell into the hands of the -rather sinister J.P. Morgan financial cartel, robber-baron -speculators who dominated Wall Street. - -At this point, history might have taken a different turn. -American might well have been served forever by a patchwork -of locally owned telephone companies. Many state politicians -and local businessmen considered this an excellent solution. - -But the new Bell holding company, American Telephone and Telegraph -or AT&T, put in a new man at the helm, a visionary industrialist -named Theodore Vail. Vail, a former Post Office manager, -understood large organizations and had an innate feeling -for the nature of large-scale communications. Vail quickly -saw to it that AT&T seized the technological edge once again. -The Pupin and Campbell "loading coil," and the deForest -"audion," are both extinct technology today, but in 1913 -they gave Vail's company the best LONG-DISTANCE lines -ever built. By controlling long-distance--the links -between, and over, and above the smaller local phone -companies--AT&T swiftly gained the whip-hand over them, -and was soon devouring them right and left. - -Vail plowed the profits back into research and development, -starting the Bell tradition of huge-scale and brilliant -industrial research. - -Technically and financially, AT&T gradually steamrollered -the opposition. Independent telephone companies never -became entirely extinct, and hundreds of them flourish today. -But Vail's AT&T became the supreme communications company. -At one point, Vail's AT&T bought Western Union itself, -the very company that had derided Bell's telephone as a "toy." -Vail thoroughly reformed Western Union's hidebound business -along his modern principles; but when the federal government -grew anxious at this centralization of power, Vail politely -gave Western Union back. - -This centralizing process was not unique. Very similar -events had happened in American steel, oil, and railroads. -But AT&T, unlike the other companies, was to remain supreme. -The monopoly robber-barons of those other industries -were humbled and shattered by government trust-busting. - -Vail, the former Post Office official, was quite willing -to accommodate the US government; in fact he would -forge an active alliance with it. AT&T would become -almost a wing of the American government, almost -another Post Office--though not quite. AT&T would -willingly submit to federal regulation, but in return, -it would use the government's regulators as its own police, -who would keep out competitors and assure the Bell -system's profits and preeminence. - -This was the second birth--the political birth--of the -American telephone system. Vail's arrangement was to -persist, with vast success, for many decades, until 1982. -His system was an odd kind of American industrial socialism. -It was born at about the same time as Leninist Communism, -and it lasted almost as long--and, it must be admitted, -to considerably better effect. - -Vail's system worked. Except perhaps for aerospace, -there has been no technology more thoroughly dominated -by Americans than the telephone. The telephone was -seen from the beginning as a quintessentially American -technology. Bell's policy, and the policy of Theodore Vail, -was a profoundly democratic policy of UNIVERSAL ACCESS. -Vail's famous corporate slogan, "One Policy, One System, -Universal Service," was a political slogan, with a very -American ring to it. - -The American telephone was not to become the specialized tool -of government or business, but a general public utility. -At first, it was true, only the wealthy could afford -private telephones, and Bell's company pursued the -business markets primarily. The American phone system -was a capitalist effort, meant to make money; it was not a charity. -But from the first, almost all communities with telephone service -had public telephones. And many stores--especially drugstores-- -offered public use of their phones. You might not own a telephone-- -but you could always get into the system, if you really needed to. - -There was nothing inevitable about this decision to make telephones -"public" and "universal." Vail's system involved a profound act -of trust in the public. This decision was a political one, -informed by the basic values of the American republic. -The situation might have been very different; -and in other countries, under other systems, -it certainly was. - -Joseph Stalin, for instance, vetoed plans for a Soviet -phone system soon after the Bolshevik revolution. -Stalin was certain that publicly accessible telephones -would become instruments of anti-Soviet counterrevolution -and conspiracy. (He was probably right.) When telephones -did arrive in the Soviet Union, they would be instruments -of Party authority, and always heavily tapped. (Alexander -Solzhenitsyn's prison-camp novel The First Circle -describes efforts to develop a phone system more suited -to Stalinist purposes.) - -France, with its tradition of rational centralized government, -had fought bitterly even against the electric telegraph, -which seemed to the French entirely too anarchical and frivolous. -For decades, nineteenth-century France communicated via the -"visual telegraph," a nation-spanning, government-owned semaphore -system of huge stone towers that signalled from hilltops, -across vast distances, with big windmill-like arms. -In 1846, one Dr. Barbay, a semaphore enthusiast, -memorably uttered an early version of what might be called -"the security expert's argument" against the open media. - -"No, the electric telegraph is not a sound invention. -It will always be at the mercy of the slightest disruption, -wild youths, drunkards, bums, etc. . . . The electric telegraph -meets those destructive elements with only a few meters of wire -over which supervision is impossible. A single man could, -without being seen, cut the telegraph wires leading to Paris, -and in twenty-four hours cut in ten different places the wires -of the same line, without being arrested. The visual telegraph, -on the contrary, has its towers, its high walls, its gates -well-guarded from inside by strong armed men. Yes, I declare, -substitution of the electric telegraph for the visual one -is a dreadful measure, a truly idiotic act." - -Dr. Barbay and his high-security stone machines -were eventually unsuccessful, but his argument-- -that communication exists for the safety and convenience -of the state, and must be carefully protected from the wild -boys and the gutter rabble who might want to crash the -system--would be heard again and again. - -When the French telephone system finally did arrive, -its snarled inadequacy was to be notorious. Devotees -of the American Bell System often recommended a trip -to France, for skeptics. - -In Edwardian Britain, issues of class and privacy -were a ball-and-chain for telephonic progress. It was -considered outrageous that anyone--any wild fool off -the street--could simply barge bellowing into one's office -or home, preceded only by the ringing of a telephone bell. -In Britain, phones were tolerated for the use of business, -but private phones tended be stuffed away into closets, -smoking rooms, or servants' quarters. Telephone operators -were resented in Britain because they did not seem to -"know their place." And no one of breeding would print -a telephone number on a business card; this seemed a crass -attempt to make the acquaintance of strangers. - -But phone access in America was to become a popular right; -something like universal suffrage, only more so. -American women could not yet vote when the phone system -came through; yet from the beginning American women -doted on the telephone. This "feminization" of the -American telephone was often commented on by foreigners. -Phones in America were not censored or stiff or formalized; -they were social, private, intimate, and domestic. -In America, Mother's Day is by far the busiest day -of the year for the phone network. - -The early telephone companies, and especially AT&T, -were among the foremost employers of American women. -They employed the daughters of the American middle-class -in great armies: in 1891, eight thousand women; by 1946, -almost a quarter of a million. Women seemed to enjoy -telephone work; it was respectable, it was steady, -it paid fairly well as women's work went, and--not least-- -it seemed a genuine contribution to the social good -of the community. Women found Vail's ideal of public -service attractive. This was especially true in rural areas, -where women operators, running extensive rural party-lines, -enjoyed considerable social power. The operator knew everyone -on the party-line, and everyone knew her. - -Although Bell himself was an ardent suffragist, the -telephone company did not employ women for the sake of -advancing female liberation. AT&T did this for sound -commercial reasons. The first telephone operators of -the Bell system were not women, but teenage American boys. -They were telegraphic messenger boys (a group about to -be rendered technically obsolescent), who swept up -around the phone office, dunned customers for bills, -and made phone connections on the switchboard, -all on the cheap. - -Within the very first year of operation, 1878, -Bell's company learned a sharp lesson about combining -teenage boys and telephone switchboards. Putting -teenage boys in charge of the phone system brought swift -and consistent disaster. Bell's chief engineer described them -as "Wild Indians." The boys were openly rude to customers. -They talked back to subscribers, saucing off, -uttering facetious remarks, and generally giving lip. -The rascals took Saint Patrick's Day off without permission. -And worst of all they played clever tricks with -the switchboard plugs: disconnecting calls, crossing lines -so that customers found themselves talking to strangers, -and so forth. - -This combination of power, technical mastery, and effective -anonymity seemed to act like catnip on teenage boys. - -This wild-kid-on-the-wires phenomenon was not confined to -the USA; from the beginning, the same was true of the British -phone system. An early British commentator kindly remarked: -"No doubt boys in their teens found the work not a little irksome, -and it is also highly probable that under the early conditions -of employment the adventurous and inquisitive spirits of which -the average healthy boy of that age is possessed, were not always -conducive to the best attention being given to the wants -of the telephone subscribers." - -So the boys were flung off the system--or at least, -deprived of control of the switchboard. But the -"adventurous and inquisitive spirits" of the teenage boys -would be heard from in the world of telephony, again and again. - -The fourth stage in the technological life-cycle is death: -"the Dog," dead tech. The telephone has so far avoided this fate. -On the contrary, it is thriving, still spreading, still evolving, -and at increasing speed. - -The telephone has achieved a rare and exalted state for a -technological artifact: it has become a HOUSEHOLD OBJECT. -The telephone, like the clock, like pen and paper, -like kitchen utensils and running water, has become -a technology that is visible only by its absence. -The telephone is technologically transparent. -The global telephone system is the largest and most -complex machine in the world, yet it is easy to use. -More remarkable yet, the telephone is almost entirely -physically safe for the user. - -For the average citizen in the 1870s, the telephone -was weirder, more shocking, more "high-tech" and -harder to comprehend, than the most outrageous stunts -of advanced computing for us Americans in the 1990s. -In trying to understand what is happening to us today, -with our bulletin-board systems, direct overseas dialling, -fiber-optic transmissions, computer viruses, hacking stunts, -and a vivid tangle of new laws and new crimes, it is important -to realize that our society has been through a similar challenge before-- -and that, all in all, we did rather well by it. - -Bell's stage telephone seemed bizarre at first. But the -sensations of weirdness vanished quickly, once people began -to hear the familiar voices of relatives and friends, -in their own homes on their own telephones. The telephone -changed from a fearsome high-tech totem to an everyday pillar -of human community. - -This has also happened, and is still happening, -to computer networks. Computer networks such as -NSFnet, BITnet, USENET, JANET, are technically -advanced, intimidating, and much harder to use than -telephones. Even the popular, commercial computer -networks, such as GEnie, Prodigy, and CompuServe, -cause much head-scratching and have been described -as "user-hateful." Nevertheless they too are changing -from fancy high-tech items into everyday sources -of human community. - -The words "community" and "communication" have -the same root. Wherever you put a communications -network, you put a community as well. And whenever -you TAKE AWAY that network--confiscate it, outlaw it, -crash it, raise its price beyond affordability-- -then you hurt that community. - -Communities will fight to defend themselves. People will fight harder -and more bitterly to defend their communities, than they will fight -to defend their own individual selves. And this is very true -of the "electronic community" that arose around computer networks -in the 1980s--or rather, the VARIOUS electronic communities, -in telephony, law enforcement, computing, and the digital -underground that, by the year 1990, were raiding, rallying, -arresting, suing, jailing, fining and issuing angry manifestos. - -None of the events of 1990 were entirely new. -Nothing happened in 1990 that did not have some kind -of earlier and more understandable precedent. What gave -the Hacker Crackdown its new sense of gravity and -importance was the feeling--the COMMUNITY feeling-- -that the political stakes had been raised; that trouble -in cyberspace was no longer mere mischief or inconclusive -skirmishing, but a genuine fight over genuine issues, -a fight for community survival and the shape of the future. - -These electronic communities, having flourished throughout -the 1980s, were becoming aware of themselves, and increasingly, -becoming aware of other, rival communities. Worries were -sprouting up right and left, with complaints, rumors, -uneasy speculations. But it would take a catalyst, a shock, -to make the new world evident. Like Bell's great publicity break, -the Tarriffville Rail Disaster of January 1878, -it would take a cause celebre. - -That cause was the AT&T Crash of January 15, 1990. -After the Crash, the wounded and anxious telephone -community would come out fighting hard. - -# - -The community of telephone technicians, engineers, operators -and researchers is the oldest community in cyberspace. -These are the veterans, the most developed group, -the richest, the most respectable, in most ways the most powerful. -Whole generations have come and gone since Alexander Graham Bell's day, -but the community he founded survives; people work for the phone system -today whose great-grandparents worked for the phone system. -Its specialty magazines, such as Telephony, AT&T Technical Journal, -Telephone Engineer and Management, are decades old; -they make computer publications like Macworld and PC Week -look like amateur johnny-come-latelies. - -And the phone companies take no back seat in high-technology, either. -Other companies' industrial researchers may have won new markets; -but the researchers of Bell Labs have won SEVEN NOBEL PRIZES. -One potent device that Bell Labs originated, the transistor, -has created entire GROUPS of industries. Bell Labs are -world-famous for generating "a patent a day," and have even -made vital discoveries in astronomy, physics and cosmology. - -Throughout its seventy-year history, "Ma Bell" was not so much -a company as a way of life. Until the cataclysmic divestiture -of the 1980s, Ma Bell was perhaps the ultimate maternalist mega-employer. -The AT&T corporate image was the "gentle giant," "the voice with a smile," -a vaguely socialist-realist world of cleanshaven linemen in shiny helmets -and blandly pretty phone-girls in headsets and nylons. Bell System -employees were famous as rock-ribbed Kiwanis and Rotary members, -Little-League enthusiasts, school-board people. - -During the long heyday of Ma Bell, the Bell employee corps -were nurtured top-to-bottom on a corporate ethos of public service. -There was good money in Bell, but Bell was not ABOUT money; -Bell used public relations, but never mere marketeering. -People went into the Bell System for a good life, -and they had a good life. But it was not mere money -that led Bell people out in the midst of storms and earthquakes -to fight with toppled phone-poles, to wade in flooded manholes, -to pull the red-eyed graveyard-shift over collapsing switching-systems. -The Bell ethic was the electrical equivalent of the postman's: -neither rain, nor snow, nor gloom of night would stop these couriers. - -It is easy to be cynical about this, as it is easy to be -cynical about any political or social system; but cynicism -does not change the fact that thousands of people took -these ideals very seriously. And some still do. - -The Bell ethos was about public service; and that was -gratifying; but it was also about private POWER, and that -was gratifying too. As a corporation, Bell was very special. -Bell was privileged. Bell had snuggled up close to the state. -In fact, Bell was as close to government as you could get in -America and still make a whole lot of legitimate money. - -But unlike other companies, Bell was above and beyond -the vulgar commercial fray. Through its regional operating companies, -Bell was omnipresent, local, and intimate, all over America; -but the central ivory towers at its corporate heart were the -tallest and the ivoriest around. - -There were other phone companies in America, to be sure; -the so-called independents. Rural cooperatives, mostly; -small fry, mostly tolerated, sometimes warred upon. -For many decades, "independent" American phone companies -lived in fear and loathing of the official Bell monopoly -(or the "Bell Octopus," as Ma Bell's nineteenth-century -enemies described her in many angry newspaper manifestos). -Some few of these independent entrepreneurs, while legally -in the wrong, fought so bitterly against the Octopus -that their illegal phone networks were cast into the street -by Bell agents and publicly burned. - -The pure technical sweetness of the Bell System gave its operators, -inventors and engineers a deeply satisfying sense of power and mastery. -They had devoted their lives to improving this vast nation-spanning machine; -over years, whole human lives, they had watched it improve and grow. -It was like a great technological temple. They were an elite, -and they knew it--even if others did not; in fact, they felt -even more powerful BECAUSE others did not understand. - -The deep attraction of this sensation of elite technical power -should never be underestimated. "Technical power" is not for everybody; -for many people it simply has no charm at all. But for some people, -it becomes the core of their lives. For a few, it is overwhelming, -obsessive; it becomes something close to an addiction. People--especially -clever teenage boys whose lives are otherwise mostly powerless and put-upon ---love this sensation of secret power, and are willing to do all sorts -of amazing things to achieve it. The technical POWER of electronics -has motivated many strange acts detailed in this book, which would -otherwise be inexplicable. - -So Bell had power beyond mere capitalism. The Bell service ethos worked, -and was often propagandized, in a rather saccharine fashion. Over the decades, -people slowly grew tired of this. And then, openly impatient with it. -By the early 1980s, Ma Bell was to find herself with scarcely a real friend -in the world. Vail's industrial socialism had become hopelessly -out-of-fashion politically. Bell would be punished for that. -And that punishment would fall harshly upon the people of the -telephone community. - -# - -In 1983, Ma Bell was dismantled by federal court action. -The pieces of Bell are now separate corporate entities. -The core of the company became AT&T Communications, -and also AT&T Industries (formerly Western Electric, -Bell's manufacturing arm). AT&T Bell Labs became Bell -Communications Research, Bellcore. Then there are the -Regional Bell Operating Companies, or RBOCs, pronounced "arbocks." - -Bell was a titan and even these regional chunks are gigantic enterprises: -Fortune 50 companies with plenty of wealth and power behind them. -But the clean lines of "One Policy, One System, Universal Service" -have been shattered, apparently forever. - -The "One Policy" of the early Reagan Administration was to -shatter a system that smacked of noncompetitive socialism. -Since that time, there has been no real telephone "policy" -on the federal level. Despite the breakup, the remnants -of Bell have never been set free to compete in the open marketplace. - -The RBOCs are still very heavily regulated, but not from the top. -Instead, they struggle politically, economically and legally, -in what seems an endless turmoil, in a patchwork of overlapping federal -and state jurisdictions. Increasingly, like other major American corporations, -the RBOCs are becoming multinational, acquiring important commercial interests -in Europe, Latin America, and the Pacific Rim. But this, too, adds to their -legal and political predicament. - -The people of what used to be Ma Bell are not happy about their fate. -They feel ill-used. They might have been grudgingly willing to make -a full transition to the free market; to become just companies amid -other companies. But this never happened. Instead, AT&T and the RBOCS -("the Baby Bells") feel themselves wrenched from side to side by state -regulators, by Congress, by the FCC, and especially by the federal court -of Judge Harold Greene, the magistrate who ordered the Bell breakup -and who has been the de facto czar of American telecommunications -ever since 1983. - -Bell people feel that they exist in a kind of paralegal limbo today. -They don't understand what's demanded of them. If it's "service," -why aren't they treated like a public service? And if it's money, -then why aren't they free to compete for it? No one seems to know, -really. Those who claim to know keep changing their minds. -Nobody in authority seems willing to grasp the nettle for once and all. - -Telephone people from other countries are amazed by the -American telephone system today. Not that it works so well; -for nowadays even the French telephone system works, more or less. -They are amazed that the American telephone system STILL works -AT ALL, under these strange conditions. - -Bell's "One System" of long-distance service is now only about -eighty percent of a system, with the remainder held by Sprint, MCI, -and the midget long-distance companies. Ugly wars over dubious -corporate practices such as "slamming" (an underhanded method -of snitching clients from rivals) break out with some regularity -in the realm of long-distance service. The battle to break Bell's -long-distance monopoly was long and ugly, and since the breakup -the battlefield has not become much prettier. AT&T's famous -shame-and-blame advertisements, which emphasized the shoddy work -and purported ethical shadiness of their competitors, were much -remarked on for their studied psychological cruelty. - -There is much bad blood in this industry, and much -long-treasured resentment. AT&T's post-breakup -corporate logo, a striped sphere, is known in the -industry as the "Death Star" (a reference from the movie -Star Wars, in which the "Death Star" was the spherical -high- tech fortress of the harsh-breathing imperial ultra-baddie, -Darth Vader.) Even AT&T employees are less than thrilled -by the Death Star. A popular (though banned) T-shirt among -AT&T employees bears the old-fashioned Bell logo of the Bell System, -plus the newfangled striped sphere, with the before-and-after comments: -"This is your brain--This is your brain on drugs!" AT&T made a very -well-financed and determined effort to break into the personal -computer market; it was disastrous, and telco computer experts -are derisively known by their competitors as "the pole-climbers." -AT&T and the Baby Bell arbocks still seem to have few friends. - -Under conditions of sharp commercial competition, a crash like -that of January 15, 1990 was a major embarrassment to AT&T. -It was a direct blow against their much-treasured reputation -for reliability. Within days of the crash AT&T's -Chief Executive Officer, Bob Allen, officially apologized, -in terms of deeply pained humility: - -"AT&T had a major service disruption last Monday. -We didn't live up to our own standards of quality, -and we didn't live up to yours. It's as simple as that. -And that's not acceptable to us. Or to you. . . . -We understand how much people have come to depend -upon AT&T service, so our AT&T Bell Laboratories scientists -and our network engineers are doing everything possible -to guard against a recurrence. . . . We know there's no way -to make up for the inconvenience this problem may have caused you." - -Mr Allen's "open letter to customers" was printed in lavish ads -all over the country: in the Wall Street Journal, USA Today, -New York Times, Los Angeles Times, Chicago Tribune, -Philadelphia Inquirer, San Francisco Chronicle Examiner, -Boston Globe, Dallas Morning News, Detroit Free Press, -Washington Post, Houston Chronicle, Cleveland Plain Dealer, -Atlanta Journal Constitution, Minneapolis Star Tribune, -St. Paul Pioneer Press Dispatch, Seattle Times/Post Intelligencer, -Tacoma News Tribune, Miami Herald, Pittsburgh Press, -St. Louis Post Dispatch, Denver Post, Phoenix Republic Gazette -and Tampa Tribune. - -In another press release, AT&T went to some pains to suggest -that this "software glitch" might have happened just as easily to MCI, -although, in fact, it hadn't. (MCI's switching software was quite different -from AT&T's--though not necessarily any safer.) AT&T also announced -their plans to offer a rebate of service on Valentine's Day to make up -for the loss during the Crash. - -"Every technical resource available, including Bell Labs -scientists and engineers, has been devoted to assuring -it will not occur again," the public was told. They were -further assured that "The chances of a recurrence are small-- -a problem of this magnitude never occurred before." - -In the meantime, however, police and corporate -security maintained their own suspicions about -"the chances of recurrence" and the real reason why -a "problem of this magnitude" had appeared, seemingly -out of nowhere. Police and security knew for a fact -that hackers of unprecedented sophistication were illegally -entering, and reprogramming, certain digital switching stations. -Rumors of hidden "viruses" and secret "logic bombs" -in the switches ran rampant in the underground, -with much chortling over AT&T's predicament, -and idle speculation over what unsung hacker genius -was responsible for it. Some hackers, including police -informants, were trying hard to finger one another -as the true culprits of the Crash. - -Telco people found little comfort in objectivity when -they contemplated these possibilities. It was just too close -to the bone for them; it was embarrassing; it hurt so much, -it was hard even to talk about. - -There has always been thieving and misbehavior in the phone system. -There has always been trouble with the rival independents, -and in the local loops. But to have such trouble in the core -of the system, the long-distance switching stations, -is a horrifying affair. To telco people, this is -all the difference between finding roaches in your kitchen -and big horrid sewer-rats in your bedroom. - -From the outside, to the average citizen, the telcos -still seem gigantic and impersonal. The American public -seems to regard them as something akin to Soviet apparats. -Even when the telcos do their best corporate-citizen routine, -subsidizing magnet high-schools and sponsoring news-shows -on public television, they seem to win little except public suspicion. - -But from the inside, all this looks very different. -There's harsh competition. A legal and political system -that seems baffled and bored, when not actively hostile -to telco interests. There's a loss of morale, a deep sensation -of having somehow lost the upper hand. Technological change -has caused a loss of data and revenue to other, newer forms -of transmission. There's theft, and new forms of theft, -of growing scale and boldness and sophistication. -With all these factors, it was no surprise to see the telcos, -large and small, break out in a litany of bitter complaint. - -In late '88 and throughout 1989, telco representatives -grew shrill in their complaints to those few American law -enforcement officials who make it their business to try to -understand what telephone people are talking about. -Telco security officials had discovered the computer- -hacker underground, infiltrated it thoroughly, -and become deeply alarmed at its growing expertise. -Here they had found a target that was not only loathsome -on its face, but clearly ripe for counterattack. - -Those bitter rivals: AT&T, MCI and Sprint--and a crowd -of Baby Bells: PacBell, Bell South, Southwestern Bell, -NYNEX, USWest, as well as the Bell research consortium Bellcore, -and the independent long-distance carrier Mid-American-- -all were to have their role in the great hacker dragnet of 1990. -After years of being battered and pushed around, the telcos had, -at least in a small way, seized the initiative again. -After years of turmoil, telcos and government officials were -once again to work smoothly in concert in defense of the System. -Optimism blossomed; enthusiasm grew on all sides; -the prospective taste of vengeance was sweet. - -# - -From the beginning--even before the crackdown had a name-- -secrecy was a big problem. There were many good reasons -for secrecy in the hacker crackdown. Hackers and code-thieves -were wily prey, slinking back to their bedrooms and basements -and destroying vital incriminating evidence at the first hint of trouble. -Furthermore, the crimes themselves were heavily technical and difficult -to describe, even to police--much less to the general public. - -When such crimes HAD been described intelligibly to the public, -in the past, that very publicity had tended to INCREASE the crimes -enormously. Telco officials, while painfully aware of the vulnerabilities -of their systems, were anxious not to publicize those weaknesses. -Experience showed them that those weaknesses, once discovered, -would be pitilessly exploited by tens of thousands of people--not only -by professional grifters and by underground hackers and phone phreaks, -but by many otherwise more-or-less honest everyday folks, who regarded -stealing service from the faceless, soulless "Phone Company" as a kind of -harmless indoor sport. When it came to protecting their interests, -telcos had long since given up on general public sympathy for -"the Voice with a Smile." Nowadays the telco's "Voice" was -very likely to be a computer's; and the American public -showed much less of the proper respect and gratitude due -the fine public service bequeathed them by Dr. Bell and Mr. Vail. -The more efficient, high-tech, computerized, and impersonal -the telcos became, it seemed, the more they were met by -sullen public resentment and amoral greed. - -Telco officials wanted to punish the phone-phreak underground, in as -public and exemplary a manner as possible. They wanted to make dire -examples of the worst offenders, to seize the ringleaders and intimidate -the small fry, to discourage and frighten the wacky hobbyists, and send -the professional grifters to jail. To do all this, publicity was vital. - -Yet operational secrecy was even more so. If word got out that -a nationwide crackdown was coming, the hackers might simply vanish; -destroy the evidence, hide their computers, go to earth, -and wait for the campaign to blow over. Even the young -hackers were crafty and suspicious, and as for the professional grifters, -they tended to split for the nearest state-line at the first sign of trouble. -For the crackdown to work well, they would all have to be caught red-handed, -swept upon suddenly, out of the blue, from every corner of the compass. - -And there was another strong motive for secrecy. In the worst-case scenario, -a blown campaign might leave the telcos open to a devastating hacker -counter-attack. If there were indeed hackers loose in America who -had caused the January 15 Crash--if there were truly gifted hackers, -loose in the nation's long-distance switching systems, and enraged -or frightened by the crackdown--then they might react unpredictably -to an attempt to collar them. Even if caught, they might have talented -and vengeful friends still running around loose. Conceivably, -it could turn ugly. Very ugly. In fact, it was hard to imagine -just how ugly things might turn, given that possibility. - -Counter-attack from hackers was a genuine concern for the telcos. -In point of fact, they would never suffer any such counter-attack. -But in months to come, they would be at some pains to publicize -this notion and to utter grim warnings about it. - -Still, that risk seemed well worth running. Better to run the risk -of vengeful attacks, than to live at the mercy of potential crashers. -Any cop would tell you that a protection racket had no real future. - -And publicity was such a useful thing. Corporate security officers, -including telco security, generally work under conditions of great discretion. -And corporate security officials do not make money for their companies. -Their job is to PREVENT THE LOSS of money, which is much less glamorous -than actually winning profits. - -If you are a corporate security official, and you do your job brilliantly, -then nothing bad happens to your company at all. Because of this, you appear -completely superfluous. This is one of the many unattractive aspects -of security work. It's rare that these folks have the chance to draw -some healthy attention to their own efforts. - -Publicity also served the interest of their friends in law enforcement. -Public officials, including law enforcement officials, thrive by attracting -favorable public interest. A brilliant prosecution in a matter of vital -public interest can make the career of a prosecuting attorney. -And for a police officer, good publicity opens the purses of the legislature; -it may bring a citation, or a promotion, or at least a rise in status -and the respect of one's peers. - -But to have both publicity and secrecy is to have one's cake and eat it too. -In months to come, as we will show, this impossible act was to cause great -pain to the agents of the crackdown. But early on, it seemed possible ---maybe even likely--that the crackdown could successfully combine -the best of both worlds. The ARREST of hackers would be heavily publicized. -The actual DEEDS of the hackers, which were technically hard to explain -and also a security risk, would be left decently obscured. The THREAT -hackers posed would be heavily trumpeted; the likelihood of their actually -committing such fearsome crimes would be left to the public's imagination. -The spread of the computer underground, and its growing technical -sophistication, would be heavily promoted; the actual hackers themselves, -mostly bespectacled middle-class white suburban teenagers, -would be denied any personal publicity. - -It does not seem to have occurred to any telco official -that the hackers accused would demand a day in court; -that journalists would smile upon the hackers as -"good copy;" that wealthy high-tech entrepreneurs would -offer moral and financial support to crackdown victims; -that constitutional lawyers would show up with briefcases, -frowning mightily. This possibility does not seem to have -ever entered the game-plan. - -And even if it had, it probably would not have slowed -the ferocious pursuit of a stolen phone-company document, -mellifluously known as "Control Office Administration of -Enhanced 911 Services for Special Services and Major Account Centers." - -In the chapters to follow, we will explore the worlds -of police and the computer underground, and the large -shadowy area where they overlap. But first, we must -explore the battleground. Before we leave the world -of the telcos, we must understand what a switching system -actually is and how your telephone actually works. - -# - -To the average citizen, the idea of the telephone is represented by, -well, a TELEPHONE: a device that you talk into. To a telco -professional, however, the telephone itself is known, in lordly -fashion, as a "subset." The "subset" in your house is a mere adjunct, -a distant nerve ending, of the central switching stations, -which are ranked in levels of heirarchy, up to the long-distance electronic -switching stations, which are some of the largest computers on earth. - -Let us imagine that it is, say, 1925, before the -introduction of computers, when the phone system was -simpler and somewhat easier to grasp. Let's further -imagine that you are Miss Leticia Luthor, a fictional -operator for Ma Bell in New York City of the 20s. - -Basically, you, Miss Luthor, ARE the "switching system." -You are sitting in front of a large vertical switchboard, -known as a "cordboard," made of shiny wooden panels, -with ten thousand metal-rimmed holes punched in them, -known as jacks. The engineers would have put more -holes into your switchboard, but ten thousand is -as many as you can reach without actually having -to get up out of your chair. - -Each of these ten thousand holes has its own little electric lightbulb, -known as a "lamp," and its own neatly printed number code. - -With the ease of long habit, you are scanning your board for lit-up bulbs. -This is what you do most of the time, so you are used to it. - -A lamp lights up. This means that the phone -at the end of that line has been taken off the hook. -Whenever a handset is taken off the hook, that closes a circuit -inside the phone which then signals the local office, i.e. you, -automatically. There might be somebody calling, or then -again the phone might be simply off the hook, but this -does not matter to you yet. The first thing you do, -is record that number in your logbook, in your fine American -public-school handwriting. This comes first, naturally, -since it is done for billing purposes. - -You now take the plug of your answering cord, which goes -directly to your headset, and plug it into the lit-up hole. -"Operator," you announce. - -In operator's classes, before taking this job, you have -been issued a large pamphlet full of canned operator's -responses for all kinds of contingencies, which you had -to memorize. You have also been trained in a proper -non-regional, non-ethnic pronunciation and tone of voice. -You rarely have the occasion to make any spontaneous -remark to a customer, and in fact this is frowned upon -(except out on the rural lines where people have time -on their hands and get up to all kinds of mischief). - -A tough-sounding user's voice at the end of the line -gives you a number. Immediately, you write that number -down in your logbook, next to the caller's number, -which you just wrote earlier. You then look and see if -the number this guy wants is in fact on your switchboard, -which it generally is, since it's generally a local call. -Long distance costs so much that people use it sparingly. - -Only then do you pick up a calling-cord from a shelf -at the base of the switchboard. This is a long elastic cord -mounted on a kind of reel so that it will zip back in when -you unplug it. There are a lot of cords down there, -and when a bunch of them are out at once they look like -a nest of snakes. Some of the girls think there are bugs -living in those cable-holes. They're called "cable mites" -and are supposed to bite your hands and give you rashes. -You don't believe this, yourself. - -Gripping the head of your calling-cord, you slip the tip -of it deftly into the sleeve of the jack for the called person. -Not all the way in, though. You just touch it. If you hear -a clicking sound, that means the line is busy and you can't -put the call through. If the line is busy, you have to stick -the calling-cord into a "busy-tone jack," which will give -the guy a busy-tone. This way you don't have to talk to him -yourself and absorb his natural human frustration. - -But the line isn't busy. So you pop the cord all the way in. -Relay circuits in your board make the distant phone ring, -and if somebody picks it up off the hook, then a phone -conversation starts. You can hear this conversation -on your answering cord, until you unplug it. In fact -you could listen to the whole conversation if you wanted, -but this is sternly frowned upon by management, and frankly, -when you've overheard one, you've pretty much heard 'em all. - -You can tell how long the conversation lasts by the glow -of the calling-cord's lamp, down on the calling-cord's shelf. -When it's over, you unplug and the calling-cord zips back into place. - -Having done this stuff a few hundred thousand times, -you become quite good at it. In fact you're plugging, -and connecting, and disconnecting, ten, twenty, forty cords -at a time. It's a manual handicraft, really, quite satisfying -in a way, rather like weaving on an upright loom. - -Should a long-distance call come up, it would be different, -but not all that different. Instead of connecting the call -through your own local switchboard, you have to go up the hierarchy, -onto the long-distance lines, known as "trunklines." -Depending on how far the call goes, it may have to work -its way through a whole series of operators, which can -take quite a while. The caller doesn't wait on the line -while this complex process is negotiated across the country -by the gaggle of operators. Instead, the caller hangs up, -and you call him back yourself when the call has finally -worked its way through. - -After four or five years of this work, you get married, -and you have to quit your job, this being the natural order -of womanhood in the American 1920s. The phone company -has to train somebody else--maybe two people, since -the phone system has grown somewhat in the meantime. -And this costs money. - -In fact, to use any kind of human being as a switching -system is a very expensive proposition. Eight thousand -Leticia Luthors would be bad enough, but a quarter of a -million of them is a military-scale proposition and makes -drastic measures in automation financially worthwhile. - -Although the phone system continues to grow today, -the number of human beings employed by telcos has -been dropping steadily for years. Phone "operators" -now deal with nothing but unusual contingencies, -all routine operations having been shrugged off onto machines. -Consequently, telephone operators are considerably less -machine-like nowadays, and have been known to have accents -and actual character in their voices. When you reach -a human operator today, the operators are rather more -"human" than they were in Leticia's day--but on the other hand, -human beings in the phone system are much harder to reach -in the first place. - -Over the first half of the twentieth century, -"electromechanical" switching systems of growing -complexity were cautiously introduced into the phone system. -In certain backwaters, some of these hybrid systems are still -in use. But after 1965, the phone system began to go completely -electronic, and this is by far the dominant mode today. -Electromechanical systems have "crossbars," and "brushes," -and other large moving mechanical parts, which, while faster -and cheaper than Leticia, are still slow, and tend to wear out -fairly quickly. - -But fully electronic systems are inscribed on silicon chips, -and are lightning-fast, very cheap, and quite durable. -They are much cheaper to maintain than even the best -electromechanical systems, and they fit into half the space. -And with every year, the silicon chip grows smaller, faster, -and cheaper yet. Best of all, automated electronics work -around the clock and don't have salaries or health insurance. - -There are, however, quite serious drawbacks to the -use of computer-chips. When they do break down, it is -a daunting challenge to figure out what the heck has gone -wrong with them. A broken cordboard generally had -a problem in it big enough to see. A broken chip has -invisible, microscopic faults. And the faults in bad -software can be so subtle as to be practically theological. - -If you want a mechanical system to do something new, -then you must travel to where it is, and pull pieces out of it, -and wire in new pieces. This costs money. However, if you want -a chip to do something new, all you have to do is change its software, -which is easy, fast and dirt-cheap. You don't even have to see the chip -to change its program. Even if you did see the chip, it wouldn't look -like much. A chip with program X doesn't look one whit different from -a chip with program Y. - -With the proper codes and sequences, and access to specialized phone-lines, -you can change electronic switching systems all over America from anywhere -you please. - -And so can other people. If they know how, and if they want to, -they can sneak into a microchip via the special phonelines and diddle with it, -leaving no physical trace at all. If they broke into the operator's station -and held Leticia at gunpoint, that would be very obvious. If they broke into -a telco building and went after an electromechanical switch with a toolbelt, -that would at least leave many traces. But people can do all manner of amazing -things to computer switches just by typing on a keyboard, and keyboards are -everywhere today. The extent of this vulnerability is deep, dark, broad, -almost mind-boggling, and yet this is a basic, primal fact of life about -any computer on a network. - -Security experts over the past twenty years have insisted, -with growing urgency, that this basic vulnerability of computers -represents an entirely new level of risk, of unknown but obviously -dire potential to society. And they are right. - -An electronic switching station does pretty much -everything Letitia did, except in nanoseconds and -on a much larger scale. Compared to Miss Luthor's -ten thousand jacks, even a primitive 1ESS switching computer, -60s vintage, has a 128,000 lines. And the current AT&T -system of choice is the monstrous fifth-generation 5ESS. - -An Electronic Switching Station can scan every line on its "board" -in a tenth of a second, and it does this over and over, tirelessly, -around the clock. Instead of eyes, it uses "ferrod scanners" -to check the condition of local lines and trunks. Instead of hands, -it has "signal distributors," "central pulse distributors," -"magnetic latching relays," and "reed switches," which complete -and break the calls. Instead of a brain, it has a "central processor." -Instead of an instruction manual, it has a program. Instead of -a handwritten logbook for recording and billing calls, -it has magnetic tapes. And it never has to talk to anybody. -Everything a customer might say to it is done by punching -the direct-dial tone buttons on your subset. - -Although an Electronic Switching Station can't talk, -it does need an interface, some way to relate to its, er, -employers. This interface is known as the "master control -center." (This interface might be better known simply as -"the interface," since it doesn't actually "control" phone -calls directly. However, a term like "Master Control -Center" is just the kind of rhetoric that telco maintenance -engineers--and hackers--find particularly satisfying.) - -Using the master control center, a phone engineer can test -local and trunk lines for malfunctions. He (rarely she) -can check various alarm displays, measure traffic on the lines, -examine the records of telephone usage and the charges for those calls, -and change the programming. - -And, of course, anybody else who gets into the master control center -by remote control can also do these things, if he (rarely she) -has managed to figure them out, or, more likely, has somehow swiped -the knowledge from people who already know. - -In 1989 and 1990, one particular RBOC, BellSouth, -which felt particularly troubled, spent a purported $1.2 -million on computer security. Some think it spent as -much as two million, if you count all the associated costs. -Two million dollars is still very little compared to the -great cost-saving utility of telephonic computer systems. - -Unfortunately, computers are also stupid. -Unlike human beings, computers possess the truly -profound stupidity of the inanimate. - -In the 1960s, in the first shocks of spreading computerization, -there was much easy talk about the stupidity of computers-- -how they could "only follow the program" and were rigidly required -to do "only what they were told." There has been rather less talk -about the stupidity of computers since they began to achieve -grandmaster status in chess tournaments, and to manifest -many other impressive forms of apparent cleverness. - -Nevertheless, computers STILL are profoundly brittle and stupid; -they are simply vastly more subtle in their stupidity and brittleness. -The computers of the 1990s are much more reliable in their components -than earlier computer systems, but they are also called upon to do -far more complex things, under far more challenging conditions. - -On a basic mathematical level, every single line of -a software program offers a chance for some possible screwup. -Software does not sit still when it works; it "runs," -it interacts with itself and with its own inputs and outputs. -By analogy, it stretches like putty into millions of possible -shapes and conditions, so many shapes that they can never -all be successfully tested, not even in the lifespan of the universe. -Sometimes the putty snaps. - -The stuff we call "software" is not like anything that human society -is used to thinking about. Software is something like a machine, -and something like mathematics, and something like language, and -something like thought, and art, and information. . . . But software -is not in fact any of those other things. The protean quality -of software is one of the great sources of its fascination. -It also makes software very powerful, very subtle, -very unpredictable, and very risky. - -Some software is bad and buggy. Some is "robust," -even "bulletproof." The best software is that which has -been tested by thousands of users under thousands of -different conditions, over years. It is then known as -"stable." This does NOT mean that the software is -now flawless, free of bugs. It generally means that there -are plenty of bugs in it, but the bugs are well-identified -and fairly well understood. - -There is simply no way to assure that software is free -of flaws. Though software is mathematical in nature, -it cannot by "proven" like a mathematical theorem; -software is more like language, with inherent ambiguities, -with different definitions, different assumptions, -different levels of meaning that can conflict. - -Human beings can manage, more or less, with -human language because we can catch the gist of it. - -Computers, despite years of effort in "artificial intelligence," -have proven spectacularly bad in "catching the gist" of anything at all. -The tiniest bit of semantic grit may still bring the mightiest computer -tumbling down. One of the most hazardous things you can do to a -computer program is try to improve it--to try to make it safer. -Software "patches" represent new, untried un-"stable" software, -which is by definition riskier. - -The modern telephone system has come to depend, -utterly and irretrievably, upon software. And the -System Crash of January 15, 1990, was caused by an -IMPROVEMENT in software. Or rather, an ATTEMPTED -improvement. - -As it happened, the problem itself--the problem per se--took this form. -A piece of telco software had been written in C language, a standard -language of the telco field. Within the C software was a -long "do. . .while" construct. The "do. . .while" construct -contained a "switch" statement. The "switch" statement contained -an "if" clause. The "if" clause contained a "break." The "break" -was SUPPOSED to "break" the "if clause." Instead, the "break" -broke the "switch" statement. - -That was the problem, the actual reason why people picking up phones -on January 15, 1990, could not talk to one another. - -Or at least, that was the subtle, abstract, cyberspatial -seed of the problem. This is how the problem manifested itself -from the realm of programming into the realm of real life. - -The System 7 software for AT&T's 4ESS switching station, -the "Generic 44E14 Central Office Switch Software," -had been extensively tested, and was considered very stable. -By the end of 1989, eighty of AT&T's switching systems -nationwide had been programmed with the new software. Cautiously, -thirty-four stations were left to run the slower, less-capable -System 6, because AT&T suspected there might be shakedown problems -with the new and unprecedently sophisticated System 7 network. - -The stations with System 7 were programmed to switch over to a backup net -in case of any problems. In mid-December 1989, however, a new high-velocity, -high-security software patch was distributed to each of the 4ESS switches -that would enable them to switch over even more quickly, making the System 7 -network that much more secure. - -Unfortunately, every one of these 4ESS switches was now in possession -of a small but deadly flaw. - -In order to maintain the network, switches must monitor -the condition of other switches--whether they are up and running, -whether they have temporarily shut down, whether they are overloaded -and in need of assistance, and so forth. The new software helped -control this bookkeeping function by monitoring the status calls -from other switches. - -It only takes four to six seconds for a troubled 4ESS switch -to rid itself of all its calls, drop everything temporarily, -and re-boot its software from scratch. Starting over from scratch -will generally rid the switch of any software problems that may have -developed in the course of running the system. Bugs that arise will -be simply wiped out by this process. It is a clever idea. This process -of automatically re-booting from scratch is known as the "normal fault -recovery routine." Since AT&T's software is in fact exceptionally stable, -systems rarely have to go into "fault recovery" in the first place; -but AT&T has always boasted of its "real world" reliability, and this -tactic is a belt-and-suspenders routine. - -The 4ESS switch used its new software to monitor its fellow switches -as they recovered from faults. As other switches came back on line -after recovery, they would send their "OK" signals to the switch. -The switch would make a little note to that effect in its "status map," -recognizing that the fellow switch was back and ready to go, -and should be sent some calls and put back to regular work. - -Unfortunately, while it was busy bookkeeping with the status map, -the tiny flaw in the brand-new software came into play. -The flaw caused the 4ESS switch to interact, subtly but drastically, -with incoming telephone calls from human users. If--and only if-- -two incoming phone-calls happened to hit the switch within a hundredth -of a second, then a small patch of data would be garbled by the flaw. - -But the switch had been programmed to monitor itself -constantly for any possible damage to its data. -When the switch perceived that its data had been somehow garbled, -then it too would go down, for swift repairs to its software. -It would signal its fellow switches not to send any more work. -It would go into the fault-recovery mode for four to six seconds. -And then the switch would be fine again, and would send out its "OK, -ready for work" signal. - -However, the "OK, ready for work" signal was the VERY THING THAT -HAD CAUSED THE SWITCH TO GO DOWN IN THE FIRST PLACE. And ALL the -System 7 switches had the same flaw in their status-map software. -As soon as they stopped to make the bookkeeping note that their fellow -switch was "OK," then they too would become vulnerable to the slight -chance that two phone-calls would hit them within a hundredth of a second. - -At approximately 2:25 P.M. EST on Monday, January 15, -one of AT&T's 4ESS toll switching systems in New York City -had an actual, legitimate, minor problem. It went into fault -recovery routines, announced "I'm going down," then announced, -"I'm back, I'm OK." And this cheery message then blasted -throughout the network to many of its fellow 4ESS switches. - -Many of the switches, at first, completely escaped trouble. -These lucky switches were not hit by the coincidence of -two phone calls within a hundredth of a second. -Their software did not fail--at first. But three switches-- -in Atlanta, St. Louis, and Detroit--were unlucky, -and were caught with their hands full. And they went down. -And they came back up, almost immediately. And they too began -to broadcast the lethal message that they, too, were "OK" again, -activating the lurking software bug in yet other switches. - -As more and more switches did have that bit of bad luck -and collapsed, the call-traffic became more and more densely -packed in the remaining switches, which were groaning -to keep up with the load. And of course, as the calls -became more densely packed, the switches were MUCH MORE LIKELY -to be hit twice within a hundredth of a second. - -It only took four seconds for a switch to get well. -There was no PHYSICAL damage of any kind to the switches, -after all. Physically, they were working perfectly. -This situation was "only" a software problem. - -But the 4ESS switches were leaping up and down every -four to six seconds, in a virulent spreading wave all over America, -in utter, manic, mechanical stupidity. They kept KNOCKING -one another down with their contagious "OK" messages. - -It took about ten minutes for the chain reaction to cripple the network. -Even then, switches would periodically luck-out and manage to resume -their normal work. Many calls--millions of them--were managing -to get through. But millions weren't. - -The switching stations that used System 6 were not directly affected. -Thanks to these old-fashioned switches, AT&T's national system avoided -complete collapse. This fact also made it clear to engineers that -System 7 was at fault. - -Bell Labs engineers, working feverishly in New Jersey, Illinois, -and Ohio, first tried their entire repertoire of standard network -remedies on the malfunctioning System 7. None of the remedies worked, -of course, because nothing like this had ever happened to any -phone system before. - -By cutting out the backup safety network entirely, -they were able to reduce the frenzy of "OK" messages -by about half. The system then began to recover, as the -chain reaction slowed. By 11:30 P.M. on Monday January -15, sweating engineers on the midnight shift breathed a -sigh of relief as the last switch cleared-up. - -By Tuesday they were pulling all the brand-new 4ESS software -and replacing it with an earlier version of System 7. - -If these had been human operators, rather than -computers at work, someone would simply have -eventually stopped screaming. It would have been -OBVIOUS that the situation was not "OK," and common -sense would have kicked in. Humans possess common sense-- -at least to some extent. Computers simply don't. - -On the other hand, computers can handle hundreds -of calls per second. Humans simply can't. If every single -human being in America worked for the phone company, -we couldn't match the performance of digital switches: -direct-dialling, three-way calling, speed-calling, call- -waiting, Caller ID, all the rest of the cornucopia -of digital bounty. Replacing computers with operators -is simply not an option any more. - -And yet we still, anachronistically, expect humans to -be running our phone system. It is hard for us -to understand that we have sacrificed huge amounts -of initiative and control to senseless yet powerful machines. -When the phones fail, we want somebody to be responsible. -We want somebody to blame. - -When the Crash of January 15 happened, the American populace -was simply not prepared to understand that enormous landslides -in cyberspace, like the Crash itself, can happen, -and can be nobody's fault in particular. It was easier to believe, -maybe even in some odd way more reassuring to believe, -that some evil person, or evil group, had done this to us. -"Hackers" had done it. With a virus. A trojan horse. -A software bomb. A dirty plot of some kind. People believed this, -responsible people. In 1990, they were looking hard for evidence -to confirm their heartfelt suspicions. - -And they would look in a lot of places. - -Come 1991, however, the outlines of an apparent new reality -would begin to emerge from the fog. - -On July 1 and 2, 1991, computer-software collapses -in telephone switching stations disrupted service in -Washington DC, Pittsburgh, Los Angeles and San Francisco. -Once again, seemingly minor maintenance problems had -crippled the digital System 7. About twelve million -people were affected in the Crash of July 1, 1991. - -Said the New York Times Service: "Telephone company executives -and federal regulators said they were not ruling out the possibility -of sabotage by computer hackers, but most seemed to think the problems -stemmed from some unknown defect in the software running the networks." - -And sure enough, within the week, a red-faced software company, -DSC Communications Corporation of Plano, Texas, owned up -to "glitches" in the "signal transfer point" software that -DSC had designed for Bell Atlantic and Pacific Bell. -The immediate cause of the July 1 Crash was a single -mistyped character: one tiny typographical flaw -in one single line of the software. One mistyped letter, -in one single line, had deprived the nation's capital of phone service. -It was not particularly surprising that this tiny flaw had escaped attention: -a typical System 7 station requires TEN MILLION lines of code. - -On Tuesday, September 17, 1991, came the most spectacular outage yet. -This case had nothing to do with software failures--at least, not directly. -Instead, a group of AT&T's switching stations in New York City had simply -run out of electrical power and shut down cold. Their back-up batteries -had failed. Automatic warning systems were supposed to warn of the loss -of battery power, but those automatic systems had failed as well. - -This time, Kennedy, La Guardia, and Newark airports -all had their voice and data communications cut. -This horrifying event was particularly ironic, as attacks -on airport computers by hackers had long been a standard -nightmare scenario, much trumpeted by computer-security -experts who feared the computer underground. There had even -been a Hollywood thriller about sinister hackers ruining -airport computers--DIE HARD II. - -Now AT&T itself had crippled airports with computer malfunctions-- -not just one airport, but three at once, some of the busiest in the world. - -Air traffic came to a standstill throughout the Greater New York area, -causing more than 500 flights to be cancelled, in a spreading wave -all over America and even into Europe. Another 500 or so flights -were delayed, affecting, all in all, about 85,000 passengers. -(One of these passengers was the chairman of the Federal -Communications Commission.) - -Stranded passengers in New York and New Jersey were further -infuriated to discover that they could not even manage to -make a long distance phone call, to explain their delay -to loved ones or business associates. Thanks to the crash, -about four and a half million domestic calls, and half a million -international calls, failed to get through. - -The September 17 NYC Crash, unlike the previous ones, -involved not a whisper of "hacker" misdeeds. On the contrary, -by 1991, AT&T itself was suffering much of the vilification -that had formerly been directed at hackers. Congressmen were grumbling. -So were state and federal regulators. And so was the press. - -For their part, ancient rival MCI took out snide full-page -newspaper ads in New York, offering their own long-distance -services for the "next time that AT&T goes down." - -"You wouldn't find a classy company like AT&T using such advertising," -protested AT&T Chairman Robert Allen, unconvincingly. Once again, -out came the full-page AT&T apologies in newspapers, apologies for -"an inexcusable culmination of both human and mechanical failure." -(This time, however, AT&T offered no discount on later calls. -Unkind critics suggested that AT&T were worried about setting any precedent -for refunding the financial losses caused by telephone crashes.) - -Industry journals asked publicly if AT&T was "asleep at the switch." -The telephone network, America's purported marvel of high-tech reliability, -had gone down three times in 18 months. Fortune magazine listed the -Crash of September 17 among the "Biggest Business Goofs of 1991," -cruelly parodying AT&T's ad campaign in an article entitled -"AT&T Wants You Back (Safely On the Ground, God Willing)." - -Why had those New York switching systems simply run out of power? -Because no human being had attended to the alarm system. -Why did the alarm systems blare automatically, -without any human being noticing? Because the three -telco technicians who SHOULD have been listening -were absent from their stations in the power-room, -on another floor of the building--attending a training class. -A training class about the alarm systems for the power room! - -"Crashing the System" was no longer "unprecedented" by late 1991. -On the contrary, it no longer even seemed an oddity. By 1991, -it was clear that all the policemen in the world could no longer -"protect" the phone system from crashes. By far the worst crashes -the system had ever had, had been inflicted, by the system, -upon ITSELF. And this time nobody was making cocksure statements -that this was an anomaly, something that would never happen again. -By 1991 the System's defenders had met their nebulous Enemy, -and the Enemy was--the System. - - - -PART TWO: THE DIGITAL UNDERGROUND - - -The date was May 9, 1990. The Pope was touring Mexico City. -Hustlers from the Medellin Cartel were trying to buy -black-market Stinger missiles in Florida. On the comics page, -Doonesbury character Andy was dying of AIDS. And then. . .a highly -unusual item whose novelty and calculated rhetoric won it -headscratching attention in newspapers all over America. - -The US Attorney's office in Phoenix, Arizona, had issued -a press release announcing a nationwide law enforcement crackdown -against "illegal computer hacking activities." The sweep was -officially known as "Operation Sundevil." - -Eight paragraphs in the press release gave the bare facts: -twenty-seven search warrants carried out on May 8, with three arrests, -and a hundred and fifty agents on the prowl in "twelve" cities across America. -(Different counts in local press reports yielded "thirteen," "fourteen," and -"sixteen" cities.) Officials estimated that criminal losses of revenue -to telephone companies "may run into millions of dollars." Credit for -the Sundevil investigations was taken by the US Secret Service, -Assistant US Attorney Tim Holtzen of Phoenix, and the Assistant -Attorney General of Arizona, Gail Thackeray. - -The prepared remarks of Garry M. Jenkins, appearing in a U.S. Department -of Justice press release, were of particular interest. Mr. Jenkins was the -Assistant Director of the US Secret Service, and the highest-ranking federal -official to take any direct public role in the hacker crackdown of 1990. - -"Today, the Secret Service is sending a clear message to those computer hackers -who have decided to violate the laws of this nation in the mistaken belief -that they can successfully avoid detection by hiding behind the relative -anonymity of their computer terminals. (. . .) "Underground groups have been -formed for the purpose of exchanging information relevant to their criminal -activities. These groups often communicate with each other through message -systems between computers called `bulletin boards.' "Our experience shows -that many computer hacker suspects are no longer misguided teenagers, -mischievously playing games with their computers in their bedrooms. -Some are now high tech computer operators using computers to engage -in unlawful conduct." - -Who were these "underground groups" and "high-tech operators?" -Where had they come from? What did they want? Who WERE they? -Were they "mischievous?" Were they dangerous? How had "misguided teenagers" -managed to alarm the United States Secret Service? And just how widespread -was this sort of thing? - -Of all the major players in the Hacker Crackdown: the phone companies, -law enforcement, the civil libertarians, and the "hackers" themselves-- -the "hackers" are by far the most mysterious, by far the hardest to -understand, by far the WEIRDEST. - -Not only are "hackers" novel in their activities, but they come -in a variety of odd subcultures, with a variety of languages, -motives and values. - -The earliest proto-hackers were probably those unsung mischievous -telegraph boys who were summarily fired by the Bell Company in 1878. - -Legitimate "hackers," those computer enthusiasts who are independent-minded -but law-abiding, generally trace their spiritual ancestry to elite technical -universities, especially M.I.T. and Stanford, in the 1960s. - -But the genuine roots of the modern hacker UNDERGROUND can probably be traced -most successfully to a now much-obscured hippie anarchist movement known as -the Yippies. The Yippies, who took their name from the largely fictional -"Youth International Party," carried out a loud and lively policy of surrealistic -subversion and outrageous political mischief. Their basic tenets were flagrant -sexual promiscuity, open and copious drug use, the political overthrow of any -powermonger over thirty years of age, and an immediate end to the war -in Vietnam, by any means necessary, including the psychic levitation -of the Pentagon. - -The two most visible Yippies were Abbie Hoffman and Jerry Rubin. -Rubin eventually became a Wall Street broker. Hoffman, ardently sought -by federal authorities, went into hiding for seven years, -in Mexico, France, and the United States. While on the lam, -Hoffman continued to write and publish, with help from sympathizers -in the American anarcho-leftist underground. Mostly, Hoffman survived -through false ID and odd jobs. Eventually he underwent facial plastic -surgery and adopted an entirely new identity as one "Barry Freed." -After surrendering himself to authorities in 1980, Hoffman spent a year -in prison on a cocaine conviction. - -Hoffman's worldview grew much darker as the glory days of the 1960s faded. -In 1989, he purportedly committed suicide, under odd and, to some, rather -suspicious circumstances. - -Abbie Hoffman is said to have caused the Federal Bureau of Investigation -to amass the single largest investigation file ever opened on an individual -American citizen. (If this is true, it is still questionable whether the -FBI regarded Abbie Hoffman a serious public threat--quite possibly, -his file was enormous simply because Hoffman left colorful legendry -wherever he went). He was a gifted publicist, who regarded electronic -media as both playground and weapon. He actively enjoyed manipulating -network TV and other gullible, image-hungry media, with various weird lies, -mindboggling rumors, impersonation scams, and other sinister distortions, -all absolutely guaranteed to upset cops, Presidential candidates, -and federal judges. Hoffman's most famous work was a book self-reflexively -known as STEAL THIS BOOK, which publicized a number of methods by which young, -penniless hippie agitators might live off the fat of a system supported by -humorless drones. STEAL THIS BOOK, whose title urged readers to damage -the very means of distribution which had put it into their hands, -might be described as a spiritual ancestor of a computer virus. - -Hoffman, like many a later conspirator, made extensive use of -pay-phones for his agitation work--in his case, generally through -the use of cheap brass washers as coin-slugs. - -During the Vietnam War, there was a federal surtax imposed on telephone -service; Hoffman and his cohorts could, and did, argue that in systematically -stealing phone service they were engaging in civil disobedience: -virtuously denying tax funds to an illegal and immoral war. - -But this thin veil of decency was soon dropped entirely. -Ripping-off the System found its own justification in deep alienation -and a basic outlaw contempt for conventional bourgeois values. -Ingenious, vaguely politicized varieties of rip-off, -which might be described as "anarchy by convenience," -became very popular in Yippie circles, and because rip-off -was so useful, it was to survive the Yippie movement itself. - -In the early 1970s, it required fairly limited expertise -and ingenuity to cheat payphones, to divert "free" -electricity and gas service, or to rob vending machines -and parking meters for handy pocket change. It also required -a conspiracy to spread this knowledge, and the gall -and nerve actually to commit petty theft, but the Yippies -had these qualifications in plenty. In June 1971, Abbie -Hoffman and a telephone enthusiast sarcastically known -as "Al Bell" began publishing a newsletter called Youth -International Party Line. This newsletter was dedicated -to collating and spreading Yippie rip-off techniques, -especially of phones, to the joy of the freewheeling -underground and the insensate rage of all straight people. -As a political tactic, phone-service theft ensured -that Yippie advocates would always have ready access -to the long-distance telephone as a medium, despite -the Yippies' chronic lack of organization, discipline, -money, or even a steady home address. - -PARTY LINE was run out of Greenwich Village for a couple of years, -then "Al Bell" more or less defected from the faltering ranks of Yippiedom, -changing the newsletter's name to TAP or Technical Assistance Program. -After the Vietnam War ended, the steam began leaking rapidly out of American -radical dissent. But by this time, "Bell" and his dozen or so -core contributors had the bit between their teeth, -and had begun to derive tremendous gut-level satisfaction -from the sensation of pure TECHNICAL POWER. - -TAP articles, once highly politicized, became pitilessly jargonized -and technical, in homage or parody to the Bell System's own technical -documents, which TAP studied closely, gutted, and reproduced without -permission. The TAP elite revelled in gloating possession -of the specialized knowledge necessary to beat the system. - -"Al Bell" dropped out of the game by the late 70s, -and "Tom Edison" took over; TAP readers (some 1400 of -them, all told) now began to show more interest in telex -switches and the growing phenomenon of computer systems. - -In 1983, "Tom Edison" had his computer stolen and his house -set on fire by an arsonist. This was an eventually mortal blow -to TAP (though the legendary name was to be resurrected -in 1990 by a young Kentuckian computer-outlaw named "Predat0r.") - -# - -Ever since telephones began to make money, there have been -people willing to rob and defraud phone companies. -The legions of petty phone thieves vastly outnumber those -"phone phreaks" who "explore the system" for the sake -of the intellectual challenge. The New York metropolitan area -(long in the vanguard of American crime) claims over 150,000 -physical attacks on pay telephones every year! Studied carefully, -a modern payphone reveals itself as a little fortress, carefully -designed and redesigned over generations, to resist coin-slugs, -zaps of electricity, chunks of coin-shaped ice, prybars, magnets, -lockpicks, blasting caps. Public pay- phones must survive in a world -of unfriendly, greedy people, and a modern payphone is as exquisitely -evolved as a cactus. -Because the phone network pre-dates the computer network, -the scofflaws known as "phone phreaks" pre-date the scofflaws -known as "computer hackers." In practice, today, the line -between "phreaking" and "hacking" is very blurred, -just as the distinction between telephones and computers -has blurred. The phone system has been digitized, -and computers have learned to "talk" over phone-lines. -What's worse--and this was the point of the Mr. Jenkins -of the Secret Service--some hackers have learned to steal, -and some thieves have learned to hack. - -Despite the blurring, one can still draw a few useful -behavioral distinctions between "phreaks" and "hackers." -Hackers are intensely interested in the "system" per se, -and enjoy relating to machines. "Phreaks" are more -social, manipulating the system in a rough-and-ready -fashion in order to get through to other human beings, -fast, cheap and under the table. - -Phone phreaks love nothing so much as "bridges," -illegal conference calls of ten or twelve chatting -conspirators, seaboard to seaboard, lasting for many hours ---and running, of course, on somebody else's tab, -preferably a large corporation's. - -As phone-phreak conferences wear on, people drop out -(or simply leave the phone off the hook, while they -sashay off to work or school or babysitting), -and new people are phoned up and invited to join in, -from some other continent, if possible. Technical trivia, -boasts, brags, lies, head-trip deceptions, weird rumors, -and cruel gossip are all freely exchanged. - -The lowest rung of phone-phreaking is the theft of telephone access codes. -Charging a phone call to somebody else's stolen number is, of course, -a pig-easy way of stealing phone service, requiring practically no -technical expertise. This practice has been very widespread, -especially among lonely people without much money who are far from home. -Code theft has flourished especially in college dorms, military bases, -and, notoriously, among roadies for rock bands. Of late, code theft -has spread very rapidly among Third Worlders in the US, who pile up -enormous unpaid long-distance bills to the Caribbean, South America, -and Pakistan. - -The simplest way to steal phone-codes is simply to look over -a victim's shoulder as he punches-in his own code-number -on a public payphone. This technique is known as "shoulder-surfing," -and is especially common in airports, bus terminals, and train stations. -The code is then sold by the thief for a few dollars. The buyer abusing -the code has no computer expertise, but calls his Mom in New York, -Kingston or Caracas and runs up a huge bill with impunity. The losses -from this primitive phreaking activity are far, far greater than the -monetary losses caused by computer-intruding hackers. - -In the mid-to-late 1980s, until the introduction of sterner telco -security measures, COMPUTERIZED code theft worked like a charm, -and was virtually omnipresent throughout the digital underground, -among phreaks and hackers alike. This was accomplished through -programming one's computer to try random code numbers over the telephone -until one of them worked. Simple programs to do this were widely available -in the underground; a computer running all night was likely to come up with -a dozen or so useful hits. This could be repeated week after week until -one had a large library of stolen codes. - -Nowadays, the computerized dialling of hundreds of numbers -can be detected within hours and swiftly traced. -If a stolen code is repeatedly abused, this too can -be detected within a few hours. But for years in the 1980s, -the publication of stolen codes was a kind of elementary etiquette -for fledgling hackers. The simplest way to establish your bona-fides -as a raider was to steal a code through repeated random dialling -and offer it to the "community" for use. Codes could be both stolen, -and used, simply and easily from the safety of one's own bedroom, -with very little fear of detection or punishment. - -Before computers and their phone-line modems entered American homes -in gigantic numbers, phone phreaks had their own special telecommunications -hardware gadget, the famous "blue box." This fraud device (now rendered -increasingly useless by the digital evolution of the phone system) could -trick switching systems into granting free access to long-distance lines. -It did this by mimicking the system's own signal, a tone of 2600 hertz. - -Steven Jobs and Steve Wozniak, the founders of Apple Computer, Inc., -once dabbled in selling blue-boxes in college dorms in California. -For many, in the early days of phreaking, blue-boxing was scarcely -perceived as "theft," but rather as a fun (if sneaky) way to use -excess phone capacity harmlessly. After all, the long-distance -lines were JUST SITTING THERE. . . . Whom did it hurt, really? -If you're not DAMAGING the system, and you're not USING UP ANY -TANGIBLE RESOURCE, and if nobody FINDS OUT what you did, -then what real harm have you done? What exactly HAVE you "stolen," -anyway? If a tree falls in the forest and nobody hears it, -how much is the noise worth? Even now this remains a rather -dicey question. - -Blue-boxing was no joke to the phone companies, however. -Indeed, when Ramparts magazine, a radical publication in California, -printed the wiring schematics necessary to create a mute box in June 1972, -the magazine was seized by police and Pacific Bell phone-company officials. -The mute box, a blue-box variant, allowed its user to receive long-distance -calls free of charge to the caller. This device was closely described in a -Ramparts article wryly titled "Regulating the Phone Company In Your Home." -Publication of this article was held to be in violation of Californian -State Penal Code section 502.7, which outlaws ownership of wire-fraud -devices and the selling of "plans or instructions for any instrument, -apparatus, or device intended to avoid telephone toll charges." - -Issues of Ramparts were recalled or seized on the newsstands, -and the resultant loss of income helped put the magazine out of business. -This was an ominous precedent for free-expression issues, but the telco's -crushing of a radical-fringe magazine passed without serious challenge -at the time. Even in the freewheeling California 1970s, it was widely felt -that there was something sacrosanct about what the phone company knew; -that the telco had a legal and moral right to protect itself by shutting -off the flow of such illicit information. Most telco information was so -"specialized" that it would scarcely be understood by any honest member -of the public. If not published, it would not be missed. To print such -material did not seem part of the legitimate role of a free press. - -In 1990 there would be a similar telco-inspired attack -on the electronic phreak/hacking "magazine" Phrack. -The Phrack legal case became a central issue in the -Hacker Crackdown, and gave rise to great controversy. -Phrack would also be shut down, for a time, at least, -but this time both the telcos and their law-enforcement -allies would pay a much larger price for their actions. -The Phrack case will be examined in detail, later. - -Phone-phreaking as a social practice is still very -much alive at this moment. Today, phone-phreaking -is thriving much more vigorously than the better-known -and worse-feared practice of "computer hacking." -New forms of phreaking are spreading rapidly, following -new vulnerabilities in sophisticated phone services. - -Cellular phones are especially vulnerable; their chips -can be re-programmed to present a false caller ID -and avoid billing. Doing so also avoids police tapping, -making cellular-phone abuse a favorite among drug-dealers. -"Call-sell operations" using pirate cellular phones can, -and have, been run right out of the backs of cars, which move -from "cell" to "cell" in the local phone system, retailing -stolen long-distance service, like some kind of demented -electronic version of the neighborhood ice-cream truck. - -Private branch-exchange phone systems in large corporations -can be penetrated; phreaks dial-up a local company, enter its -internal phone-system, hack it, then use the company's own -PBX system to dial back out over the public network, -causing the company to be stuck with the resulting -long-distance bill. This technique is known as "diverting." -"Diverting" can be very costly, especially because phreaks -tend to travel in packs and never stop talking. -Perhaps the worst by-product of this "PBX fraud" -is that victim companies and telcos have sued one another -over the financial responsibility for the stolen calls, -thus enriching not only shabby phreaks but well-paid lawyers. - -"Voice-mail systems" can also be abused; phreaks -can seize their own sections of these sophisticated -electronic answering machines, and use them for trading -codes or knowledge of illegal techniques. Voice-mail -abuse does not hurt the company directly, but finding -supposedly empty slots in your company's answering -machine all crammed with phreaks eagerly chattering -and hey-duding one another in impenetrable jargon can -cause sensations of almost mystical repulsion and dread. - -Worse yet, phreaks have sometimes been known to react -truculently to attempts to "clean up" the voice-mail system. -Rather than humbly acquiescing to being thrown out of their playground, -they may very well call up the company officials at work (or at home) -and loudly demand free voice-mail addresses of their very own. -Such bullying is taken very seriously by spooked victims. - -Acts of phreak revenge against straight people are rare, -but voice-mail systems are especially tempting and vulnerable, -and an infestation of angry phreaks in one's voice-mail system is no joke. -They can erase legitimate messages; or spy on private messages; -or harass users with recorded taunts and obscenities. -They've even been known to seize control of voice-mail security, -and lock out legitimate users, or even shut down the system entirely. - -Cellular phone-calls, cordless phones, and ship-to-shore -telephony can all be monitored by various forms of radio; -this kind of "passive monitoring" is spreading explosively today. -Technically eavesdropping on other people's cordless and cellular -phone-calls is the fastest-growing area in phreaking today. -This practice strongly appeals to the lust for power and conveys -gratifying sensations of technical superiority over the eavesdropping -victim. Monitoring is rife with all manner of tempting evil mischief. -Simple prurient snooping is by far the most common activity. -But credit-card numbers unwarily spoken over the phone can be recorded, -stolen and used. And tapping people's phone-calls (whether through -active telephone taps or passive radio monitors) does lend itself -conveniently to activities like blackmail, industrial espionage, -and political dirty tricks. - -It should be repeated that telecommunications fraud, -the theft of phone service, causes vastly greater monetary -losses than the practice of entering into computers by stealth. -Hackers are mostly young suburban American white males, -and exist in their hundreds--but "phreaks" come from both sexes -and from many nationalities, ages and ethnic backgrounds, -and are flourishing in the thousands. - -# - -The term "hacker" has had an unfortunate history. -This book, The Hacker Crackdown, has little to say about -"hacking" in its finer, original sense. The term can signify -the free-wheeling intellectual exploration of the highest -and deepest potential of computer systems. Hacking can -describe the determination to make access to computers -and information as free and open as possible. Hacking -can involve the heartfelt conviction that beauty can -be found in computers, that the fine aesthetic in a perfect -program can liberate the mind and spirit. This is "hacking" -as it was defined in Steven Levy's much-praised history -of the pioneer computer milieu, Hackers, published in 1984. - -Hackers of all kinds are absolutely soaked through with heroic -anti-bureaucratic sentiment. Hackers long for recognition -as a praiseworthy cultural archetype, the postmodern electronic -equivalent of the cowboy and mountain man. Whether they deserve -such a reputation is something for history to decide. But many hackers-- -including those outlaw hackers who are computer intruders, and whose -activities are defined as criminal--actually attempt to LIVE UP TO -this techno-cowboy reputation. And given that electronics and -telecommunications are still largely unexplored territories, -there is simply NO TELLING what hackers might uncover. - -For some people, this freedom is the very breath of oxygen, -the inventive spontaneity that makes life worth living -and that flings open doors to marvellous possibility and -individual empowerment. But for many people ---and increasingly so--the hacker is an ominous figure, -a smart-aleck sociopath ready to burst out of his basement -wilderness and savage other people's lives for his own -anarchical convenience. - -Any form of power without responsibility, without direct -and formal checks and balances, is frightening to people-- -and reasonably so. It should be frankly admitted that -hackers ARE frightening, and that the basis of this fear -is not irrational. - -Fear of hackers goes well beyond the fear of merely criminal activity. - -Subversion and manipulation of the phone system -is an act with disturbing political overtones. -In America, computers and telephones are potent symbols -of organized authority and the technocratic business elite. - -But there is an element in American culture that -has always strongly rebelled against these symbols; -rebelled against all large industrial computers -and all phone companies. A certain anarchical tinge deep -in the American soul delights in causing confusion and pain -to all bureaucracies, including technological ones. - -There is sometimes malice and vandalism in this attitude, -but it is a deep and cherished part of the American national character. -The outlaw, the rebel, the rugged individual, the pioneer, -the sturdy Jeffersonian yeoman, the private citizen resisting -interference in his pursuit of happiness--these are figures that all -Americans recognize, and that many will strongly applaud and defend. - -Many scrupulously law-abiding citizens today do cutting-edge work -with electronics--work that has already had tremendous social influence -and will have much more in years to come. In all truth, these talented, -hardworking, law-abiding, mature, adult people are far more disturbing -to the peace and order of the current status quo than any scofflaw group -of romantic teenage punk kids. These law-abiding hackers have the power, -ability, and willingness to influence other people's lives quite unpredictably. -They have means, motive, and opportunity to meddle drastically with the -American social order. When corralled into governments, universities, -or large multinational companies, and forced to follow rulebooks -and wear suits and ties, they at least have some conventional halters -on their freedom of action. But when loosed alone, or in small groups, -and fired by imagination and the entrepreneurial spirit, they can move -mountains--causing landslides that will likely crash directly into your -office and living room. - -These people, as a class, instinctively recognize that a public, -politicized attack on hackers will eventually spread to them-- -that the term "hacker," once demonized, might be used to knock -their hands off the levers of power and choke them out of existence. -There are hackers today who fiercely and publicly resist any besmirching -of the noble title of hacker. Naturally and understandably, they deeply -resent the attack on their values implicit in using the word "hacker" -as a synonym for computer-criminal. - -This book, sadly but in my opinion unavoidably, rather adds -to the degradation of the term. It concerns itself mostly with "hacking" -in its commonest latter-day definition, i.e., intruding into computer -systems by stealth and without permission. The term "hacking" is used -routinely today by almost all law enforcement officials with any -professional interest in computer fraud and abuse. American police -describe almost any crime committed with, by, through, or against -a computer as hacking. - -Most importantly, "hacker" is what computer-intruders -choose to call THEMSELVES. Nobody who "hacks" into systems -willingly describes himself (rarely, herself) as a "computer intruder," -"computer trespasser," "cracker," "wormer," "darkside hacker" -or "high tech street gangster." Several other demeaning terms -have been invented in the hope that the press and public -will leave the original sense of the word alone. But few people -actually use these terms. (I exempt the term "cyberpunk," -which a few hackers and law enforcement people actually do use. -The term "cyberpunk" is drawn from literary criticism and has -some odd and unlikely resonances, but, like hacker, -cyberpunk too has become a criminal pejorative today.) - -In any case, breaking into computer systems was hardly alien -to the original hacker tradition. The first tottering systems -of the 1960s required fairly extensive internal surgery merely -to function day-by-day. Their users "invaded" the deepest, -most arcane recesses of their operating software almost -as a matter of routine. "Computer security" in these early, -primitive systems was at best an afterthought. What security -there was, was entirely physical, for it was assumed that -anyone allowed near this expensive, arcane hardware would be -a fully qualified professional expert. - -In a campus environment, though, this meant that grad students, -teaching assistants, undergraduates, and eventually, -all manner of dropouts and hangers-on ended up accessing -and often running the works. - -Universities, even modern universities, are not in -the business of maintaining security over information. -On the contrary, universities, as institutions, pre-date -the "information economy" by many centuries and are not- -for-profit cultural entities, whose reason for existence -(purportedly) is to discover truth, codify it through -techniques of scholarship, and then teach it. Universities -are meant to PASS THE TORCH OF CIVILIZATION, not just -download data into student skulls, and the values of the -academic community are strongly at odds with those of all -would-be information empires. Teachers at all levels, from -kindergarten up, have proven to be shameless and persistent -software and data pirates. Universities do not merely -"leak information" but vigorously broadcast free thought. - -This clash of values has been fraught with controversy. -Many hackers of the 1960s remember their professional -apprenticeship as a long guerilla war against the uptight -mainframe-computer "information priesthood." These computer-hungry -youngsters had to struggle hard for access to computing power, -and many of them were not above certain, er, shortcuts. -But, over the years, this practice freed computing -from the sterile reserve of lab-coated technocrats and -was largely responsible for the explosive growth of computing -in general society--especially PERSONAL computing. - -Access to technical power acted like catnip on certain -of these youngsters. Most of the basic techniques of -computer intrusion: password cracking, trapdoors, backdoors, -trojan horses--were invented in college environments in the 1960s, -in the early days of network computing. Some off-the-cuff -experience at computer intrusion was to be in the informal -resume of most "hackers" and many future industry giants. -Outside of the tiny cult of computer enthusiasts, few people -thought much about the implications of "breaking into" -computers. This sort of activity had not yet been publicized, -much less criminalized. - -In the 1960s, definitions of "property" and "privacy" -had not yet been extended to cyberspace. Computers -were not yet indispensable to society. There were no vast -databanks of vulnerable, proprietary information stored -in computers, which might be accessed, copied without -permission, erased, altered, or sabotaged. The stakes -were low in the early days--but they grew every year, -exponentially, as computers themselves grew. - -By the 1990s, commercial and political pressures -had become overwhelming, and they broke the social -boundaries of the hacking subculture. Hacking -had become too important to be left to the hackers. -Society was now forced to tackle the intangible nature -of cyberspace-as-property, cyberspace as privately-owned -unreal-estate. In the new, severe, responsible, high-stakes -context of the "Information Society" of the 1990s, -"hacking" was called into question. - -What did it mean to break into a computer without -permission and use its computational power, or look -around inside its files without hurting anything? -What were computer-intruding hackers, anyway--how should -society, and the law, best define their actions? -Were they just BROWSERS, harmless intellectual explorers? -Were they VOYEURS, snoops, invaders of privacy? Should -they be sternly treated as potential AGENTS OF ESPIONAGE, -or perhaps as INDUSTRIAL SPIES? Or were they best -defined as TRESPASSERS, a very common teenage -misdemeanor? Was hacking THEFT OF SERVICE? -(After all, intruders were getting someone else's -computer to carry out their orders, without permission -and without paying). Was hacking FRAUD? Maybe it was -best described as IMPERSONATION. The commonest mode -of computer intrusion was (and is) to swipe or snoop -somebody else's password, and then enter the computer -in the guise of another person--who is commonly stuck -with the blame and the bills. - -Perhaps a medical metaphor was better--hackers should -be defined as "sick," as COMPUTER ADDICTS unable -to control their irresponsible, compulsive behavior. - -But these weighty assessments meant little to the -people who were actually being judged. From inside -the underground world of hacking itself, all these -perceptions seem quaint, wrongheaded, stupid, or meaningless. -The most important self-perception of underground hackers-- -from the 1960s, right through to the present day--is that -they are an ELITE. The day-to-day struggle in the underground -is not over sociological definitions--who cares?--but for power, -knowledge, and status among one's peers. - -When you are a hacker, it is your own inner conviction -of your elite status that enables you to break, or let -us say "transcend," the rules. It is not that ALL rules -go by the board. The rules habitually broken by hackers -are UNIMPORTANT rules--the rules of dopey greedhead telco -bureaucrats and pig-ignorant government pests. - -Hackers have their OWN rules, which separate behavior -which is cool and elite, from behavior which is rodentlike, -stupid and losing. These "rules," however, are mostly unwritten -and enforced by peer pressure and tribal feeling. Like all rules -that depend on the unspoken conviction that everybody else -is a good old boy, these rules are ripe for abuse. The mechanisms -of hacker peer- pressure, "teletrials" and ostracism, are rarely used -and rarely work. Back-stabbing slander, threats, and electronic -harassment are also freely employed in down-and-dirty intrahacker feuds, -but this rarely forces a rival out of the scene entirely. The only real -solution for the problem of an utterly losing, treacherous and rodentlike -hacker is to TURN HIM IN TO THE POLICE. Unlike the Mafia or Medellin Cartel, -the hacker elite cannot simply execute the bigmouths, creeps and troublemakers -among their ranks, so they turn one another in with astonishing frequency. - -There is no tradition of silence or OMERTA in the hacker underworld. -Hackers can be shy, even reclusive, but when they do talk, hackers -tend to brag, boast and strut. Almost everything hackers do is INVISIBLE; -if they don't brag, boast, and strut about it, then NOBODY WILL EVER KNOW. -If you don't have something to brag, boast, and strut about, then nobody -in the underground will recognize you and favor you with vital cooperation -and respect. - -The way to win a solid reputation in the underground -is by telling other hackers things that could only -have been learned by exceptional cunning and stealth. -Forbidden knowledge, therefore, is the basic currency -of the digital underground, like seashells among -Trobriand Islanders. Hackers hoard this knowledge, -and dwell upon it obsessively, and refine it, -and bargain with it, and talk and talk about it. - -Many hackers even suffer from a strange obsession to TEACH-- -to spread the ethos and the knowledge of the digital underground. -They'll do this even when it gains them no particular advantage -and presents a grave personal risk. - -And when that risk catches up with them, they will go right on teaching -and preaching--to a new audience this time, their interrogators from law -enforcement. Almost every hacker arrested tells everything he knows-- -all about his friends, his mentors, his disciples--legends, threats, -horror stories, dire rumors, gossip, hallucinations. This is, of course, -convenient for law enforcement--except when law enforcement begins -to believe hacker legendry. - -Phone phreaks are unique among criminals in their willingness -to call up law enforcement officials--in the office, at their homes-- -and give them an extended piece of their mind. It is hard not to -interpret this as BEGGING FOR ARREST, and in fact it is an act -of incredible foolhardiness. Police are naturally nettled -by these acts of chutzpah and will go well out of their way -to bust these flaunting idiots. But it can also be interpreted -as a product of a world-view so elitist, so closed and hermetic, -that electronic police are simply not perceived as "police," -but rather as ENEMY PHONE PHREAKS who should be scolded -into behaving "decently." - -Hackers at their most grandiloquent perceive themselves -as the elite pioneers of a new electronic world. -Attempts to make them obey the democratically -established laws of contemporary American society are -seen as repression and persecution. After all, they argue, -if Alexander Graham Bell had gone along with the rules -of the Western Union telegraph company, there would have -been no telephones. If Jobs and Wozniak had believed -that IBM was the be-all and end-all, there would have -been no personal computers. If Benjamin Franklin and -Thomas Jefferson had tried to "work within the system" -there would have been no United States. - -Not only do hackers privately believe this as an article of faith, -but they have been known to write ardent manifestos about it. -Here are some revealing excerpts from an especially vivid hacker manifesto: -"The Techno-Revolution" by "Dr. Crash," which appeared in electronic -form in Phrack Volume 1, Issue 6, Phile 3. - - -"To fully explain the true motives behind hacking, -we must first take a quick look into the past. In the 1960s, -a group of MIT students built the first modern computer system. -This wild, rebellious group of young men were the first to bear -the name `hackers.' The systems that they developed were intended -to be used to solve world problems and to benefit all of mankind. -"As we can see, this has not been the case. The computer system -has been solely in the hands of big businesses and the government. -The wonderful device meant to enrich life has become a weapon which -dehumanizes people. To the government and large businesses, -people are no more than disk space, and the government doesn't -use computers to arrange aid for the poor, but to control nuclear -death weapons. The average American can only have access -to a small microcomputer which is worth only a fraction -of what they pay for it. The businesses keep the -true state-of-the-art equipment away from the people -behind a steel wall of incredibly high prices and bureaucracy. -It is because of this state of affairs that hacking was born. (. . .) -"Of course, the government doesn't want the monopoly of technology broken, -so they have outlawed hacking and arrest anyone who is caught. (. . .) -The phone company is another example of technology abused and kept -from people with high prices. (. . .) "Hackers often find that their -existing equipment, due to the monopoly tactics of computer companies, -is inefficient for their purposes. Due to the exorbitantly high prices, -it is impossible to legally purchase the necessary equipment. -This need has given still another segment of the fight: Credit Carding. -Carding is a way of obtaining the necessary goods without paying for them. -It is again due to the companies' stupidity that Carding is so easy, -and shows that the world's businesses are in the hands of those -with considerably less technical know-how than we, the hackers. (. . .) -"Hacking must continue. We must train newcomers to the art of hacking. -(. . . .) And whatever you do, continue the fight. Whether you know it -or not, if you are a hacker, you are a revolutionary. Don't worry, -you're on the right side." - -The defense of "carding" is rare. Most hackers regard credit-card -theft as "poison" to the underground, a sleazy and immoral effort that, -worse yet, is hard to get away with. Nevertheless, manifestos advocating -credit-card theft, the deliberate crashing of computer systems, -and even acts of violent physical destruction such as vandalism -and arson do exist in the underground. These boasts and threats -are taken quite seriously by the police. And not every hacker -is an abstract, Platonic computer-nerd. Some few are quite experienced -at picking locks, robbing phone-trucks, and breaking and entering buildings. - -Hackers vary in their degree of hatred for authority -and the violence of their rhetoric. But, at a bottom line, -they are scofflaws. They don't regard the current rules -of electronic behavior as respectable efforts to preserve -law and order and protect public safety. They regard these -laws as immoral efforts by soulless corporations to protect -their profit margins and to crush dissidents. "Stupid" people, -including police, businessmen, politicians, and journalists, -simply have no right to judge the actions of those possessed of genius, -techno-revolutionary intentions, and technical expertise. - -# - -Hackers are generally teenagers and college kids not -engaged in earning a living. They often come from fairly -well-to-do middle-class backgrounds, and are markedly -anti-materialistic (except, that is, when it comes to -computer equipment). Anyone motivated by greed for -mere money (as opposed to the greed for power, -knowledge and status) is swiftly written-off as a narrow- -minded breadhead whose interests can only be corrupt -and contemptible. Having grown up in the 1970s and -1980s, the young Bohemians of the digital underground -regard straight society as awash in plutocratic corruption, -where everyone from the President down is for sale and -whoever has the gold makes the rules. - -Interestingly, there's a funhouse-mirror image of this attitude -on the other side of the conflict. The police are also -one of the most markedly anti-materialistic groups -in American society, motivated not by mere money -but by ideals of service, justice, esprit-de-corps, -and, of course, their own brand of specialized knowledge -and power. Remarkably, the propaganda war between cops -and hackers has always involved angry allegations -that the other side is trying to make a sleazy buck. -Hackers consistently sneer that anti-phreak prosecutors -are angling for cushy jobs as telco lawyers and that -computer-crime police are aiming to cash in later -as well-paid computer-security consultants in the private sector. - -For their part, police publicly conflate all -hacking crimes with robbing payphones with crowbars. -Allegations of "monetary losses" from computer intrusion -are notoriously inflated. The act of illicitly copying -a document from a computer is morally equated with -directly robbing a company of, say, half a million dollars. -The teenage computer intruder in possession of this "proprietary" -document has certainly not sold it for such a sum, would likely -have little idea how to sell it at all, and quite probably -doesn't even understand what he has. He has not made a cent -in profit from his felony but is still morally equated with -a thief who has robbed the church poorbox and lit out for Brazil. - -Police want to believe that all hackers are thieves. -It is a tortuous and almost unbearable act for the American -justice system to put people in jail because they want -to learn things which are forbidden for them to know. -In an American context, almost any pretext for punishment -is better than jailing people to protect certain restricted -kinds of information. Nevertheless, POLICING INFORMATION -is part and parcel of the struggle against hackers. - -This dilemma is well exemplified by the remarkable -activities of "Emmanuel Goldstein," editor and publisher -of a print magazine known as 2600: The Hacker Quarterly. -Goldstein was an English major at Long Island's State University -of New York in the '70s, when he became involved with the local -college radio station. His growing interest in electronics -caused him to drift into Yippie TAP circles and thus into -the digital underground, where he became a self-described -techno-rat. His magazine publishes techniques of computer -intrusion and telephone "exploration" as well as gloating -exposes of telco misdeeds and governmental failings. - -Goldstein lives quietly and very privately in a large, -crumbling Victorian mansion in Setauket, New York. -The seaside house is decorated with telco decals, chunks of -driftwood, and the basic bric-a-brac of a hippie crash-pad. -He is unmarried, mildly unkempt, and survives mostly -on TV dinners and turkey-stuffing eaten straight out -of the bag. Goldstein is a man of considerable charm -and fluency, with a brief, disarming smile and the kind -of pitiless, stubborn, thoroughly recidivist integrity -that America's electronic police find genuinely alarming. - -Goldstein took his nom-de-plume, or "handle," from -a character in Orwell's 1984, which may be taken, -correctly, as a symptom of the gravity of his sociopolitical -worldview. He is not himself a practicing computer -intruder, though he vigorously abets these actions, -especially when they are pursued against large -corporations or governmental agencies. Nor is he a thief, -for he loudly scorns mere theft of phone service, in favor -of "exploring and manipulating the system." He is probably -best described and understood as a DISSIDENT. - -Weirdly, Goldstein is living in modern America -under conditions very similar to those of former -East European intellectual dissidents. In other words, -he flagrantly espouses a value-system that is deeply -and irrevocably opposed to the system of those in power -and the police. The values in 2600 are generally expressed -in terms that are ironic, sarcastic, paradoxical, or just -downright confused. But there's no mistaking their -radically anti-authoritarian tenor. 2600 holds that -technical power and specialized knowledge, of any kind -obtainable, belong by right in the hands of those individuals -brave and bold enough to discover them--by whatever means necessary. -Devices, laws, or systems that forbid access, and the free -spread of knowledge, are provocations that any free -and self-respecting hacker should relentlessly attack. -The "privacy" of governments, corporations and other soulless -technocratic organizations should never be protected -at the expense of the liberty and free initiative -of the individual techno-rat. - -However, in our contemporary workaday world, both governments -and corporations are very anxious indeed to police information -which is secret, proprietary, restricted, confidential, -copyrighted, patented, hazardous, illegal, unethical, -embarrassing, or otherwise sensitive. This makes Goldstein -persona non grata, and his philosophy a threat. - -Very little about the conditions of Goldstein's daily -life would astonish, say, Vaclav Havel. (We may note -in passing that President Havel once had his word-processor -confiscated by the Czechoslovak police.) Goldstein lives -by SAMIZDAT, acting semi-openly as a data-center -for the underground, while challenging the powers-that-be -to abide by their own stated rules: freedom of speech -and the First Amendment. - -Goldstein thoroughly looks and acts the part of techno-rat, -with shoulder-length ringlets and a piratical black -fisherman's-cap set at a rakish angle. He often shows up -like Banquo's ghost at meetings of computer professionals, -where he listens quietly, half-smiling and taking thorough notes. - -Computer professionals generally meet publicly, -and find it very difficult to rid themselves of Goldstein -and his ilk without extralegal and unconstitutional actions. -Sympathizers, many of them quite respectable people -with responsible jobs, admire Goldstein's attitude and -surreptitiously pass him information. An unknown but -presumably large proportion of Goldstein's 2,000-plus -readership are telco security personnel and police, -who are forced to subscribe to 2600 to stay abreast -of new developments in hacking. They thus find themselves -PAYING THIS GUY'S RENT while grinding their teeth in anguish, -a situation that would have delighted Abbie Hoffman -(one of Goldstein's few idols). - -Goldstein is probably the best-known public representative -of the hacker underground today, and certainly the best-hated. -Police regard him as a Fagin, a corrupter of youth, and speak -of him with untempered loathing. He is quite an accomplished gadfly. -After the Martin Luther King Day Crash of 1990, Goldstein, -for instance, adeptly rubbed salt into the wound in the pages of 2600. -"Yeah, it was fun for the phone phreaks as we watched the network crumble," -he admitted cheerfully. "But it was also an ominous sign of what's -to come. . . . Some AT&T people, aided by well-meaning but ignorant media, -were spreading the notion that many companies had the same software -and therefore could face the same problem someday. Wrong. This was -entirely an AT&T software deficiency. Of course, other companies could -face entirely DIFFERENT software problems. But then, so too could AT&T." - -After a technical discussion of the system's failings, -the Long Island techno-rat went on to offer thoughtful -criticism to the gigantic multinational's hundreds of -professionally qualified engineers. "What we don't know -is how a major force in communications like AT&T could -be so sloppy. What happened to backups? Sure, -computer systems go down all the time, but people -making phone calls are not the same as people logging -on to computers. We must make that distinction. It's not -acceptable for the phone system or any other essential -service to `go down.' If we continue to trust technology -without understanding it, we can look forward to many -variations on this theme. - -"AT&T owes it to its customers to be prepared to INSTANTLY -switch to another network if something strange and unpredictable -starts occurring. The news here isn't so much the failure -of a computer program, but the failure of AT&T's entire structure." - -The very idea of this. . . . this PERSON. . . . offering -"advice" about "AT&T's entire structure" is more than -some people can easily bear. How dare this near-criminal -dictate what is or isn't "acceptable" behavior from AT&T? -Especially when he's publishing, in the very same issue, -detailed schematic diagrams for creating various switching-network -signalling tones unavailable to the public. - -"See what happens when you drop a `silver box' tone or two -down your local exchange or through different long distance -service carriers," advises 2600 contributor "Mr. Upsetter" -in "How To Build a Signal Box." "If you experiment systematically -and keep good records, you will surely discover something interesting." - -This is, of course, the scientific method, generally regarded -as a praiseworthy activity and one of the flowers of modern civilization. -One can indeed learn a great deal with this sort of structured -intellectual activity. Telco employees regard this mode of "exploration" -as akin to flinging sticks of dynamite into their pond to see what lives -on the bottom. - -2600 has been published consistently since 1984. -It has also run a bulletin board computer system, -printed 2600 T-shirts, taken fax calls. . . . -The Spring 1991 issue has an interesting announcement on page 45: -"We just discovered an extra set of wires attached to our fax line -and heading up the pole. (They've since been clipped.) -Your faxes to us and to anyone else could be monitored." -In the worldview of 2600, the tiny band of techno-rat brothers -(rarely, sisters) are a beseiged vanguard of the truly free and honest. -The rest of the world is a maelstrom of corporate crime and high-level -governmental corruption, occasionally tempered with well-meaning -ignorance. To read a few issues in a row is to enter a nightmare -akin to Solzhenitsyn's, somewhat tempered by the fact that 2600 -is often extremely funny. - -Goldstein did not become a target of the Hacker Crackdown, -though he protested loudly, eloquently, and publicly about it, -and it added considerably to his fame. It was not that he is not -regarded as dangerous, because he is so regarded. Goldstein has had -brushes with the law in the past: in 1985, a 2600 bulletin board -computer was seized by the FBI, and some software on it was formally -declared "a burglary tool in the form of a computer program." -But Goldstein escaped direct repression in 1990, because his -magazine is printed on paper, and recognized as subject -to Constitutional freedom of the press protection. -As was seen in the Ramparts case, this is far from -an absolute guarantee. Still, as a practical matter, -shutting down 2600 by court-order would create so much -legal hassle that it is simply unfeasible, at least -for the present. Throughout 1990, both Goldstein -and his magazine were peevishly thriving. - -Instead, the Crackdown of 1990 would concern itself -with the computerized version of forbidden data. -The crackdown itself, first and foremost, was about -BULLETIN BOARD SYSTEMS. Bulletin Board Systems, most often -known by the ugly and un-pluralizable acronym "BBS," are -the life-blood of the digital underground. Boards were -also central to law enforcement's tactics and strategy -in the Hacker Crackdown. - -A "bulletin board system" can be formally defined as -a computer which serves as an information and message- -passing center for users dialing-up over the phone-lines -through the use of modems. A "modem," or modulator- -demodulator, is a device which translates the digital -impulses of computers into audible analog telephone -signals, and vice versa. Modems connect computers -to phones and thus to each other. - -Large-scale mainframe computers have been connected since the 1960s, -but PERSONAL computers, run by individuals out of their homes, -were first networked in the late 1970s. The "board" created -by Ward Christensen and Randy Suess in February 1978, -in Chicago, Illinois, is generally regarded as the first -personal-computer bulletin board system worthy of the name. - -Boards run on many different machines, employing many -different kinds of software. Early boards were crude and buggy, -and their managers, known as "system operators" or "sysops," -were hard-working technical experts who wrote their own software. -But like most everything else in the world of electronics, -boards became faster, cheaper, better-designed, and generally -far more sophisticated throughout the 1980s. They also moved -swiftly out of the hands of pioneers and into those of the -general public. By 1985 there were something in the -neighborhood of 4,000 boards in America. By 1990 it was -calculated, vaguely, that there were about 30,000 boards in -the US, with uncounted thousands overseas. - -Computer bulletin boards are unregulated enterprises. -Running a board is a rough-and-ready, catch-as-catch-can proposition. -Basically, anybody with a computer, modem, software and a phone-line -can start a board. With second-hand equipment and public-domain -free software, the price of a board might be quite small-- -less than it would take to publish a magazine or even a -decent pamphlet. Entrepreneurs eagerly sell bulletin-board -software, and will coach nontechnical amateur sysops in its use. - -Boards are not "presses." They are not magazines, -or libraries, or phones, or CB radios, or traditional cork -bulletin boards down at the local laundry, though they -have some passing resemblance to those earlier media. -Boards are a new medium--they may even be a LARGE NUMBER of new media. - -Consider these unique characteristics: boards are cheap, -yet they can have a national, even global reach. -Boards can be contacted from anywhere in the global -telephone network, at NO COST to the person running the board-- -the caller pays the phone bill, and if the caller is local, -the call is free. Boards do not involve an editorial elite -addressing a mass audience. The "sysop" of a board is not -an exclusive publisher or writer--he is managing an electronic salon, -where individuals can address the general public, play the part -of the general public, and also exchange private mail -with other individuals. And the "conversation" on boards, -though fluid, rapid, and highly interactive, is not spoken, -but written. It is also relatively anonymous, sometimes completely so. - -And because boards are cheap and ubiquitous, regulations -and licensing requirements would likely be practically unenforceable. -It would almost be easier to "regulate," "inspect," and "license" -the content of private mail--probably more so, since the mail system -is operated by the federal government. Boards are run by individuals, -independently, entirely at their own whim. - -For the sysop, the cost of operation is not the primary -limiting factor. Once the investment in a computer and -modem has been made, the only steady cost is the charge -for maintaining a phone line (or several phone lines). -The primary limits for sysops are time and energy. -Boards require upkeep. New users are generally "validated"-- -they must be issued individual passwords, and called at -home by voice-phone, so that their identity can be -verified. Obnoxious users, who exist in plenty, must be -chided or purged. Proliferating messages must be deleted -when they grow old, so that the capacity of the system -is not overwhelmed. And software programs (if such things -are kept on the board) must be examined for possible -computer viruses. If there is a financial charge to use -the board (increasingly common, especially in larger and -fancier systems) then accounts must be kept, and users -must be billed. And if the board crashes--a very common -occurrence--then repairs must be made. - -Boards can be distinguished by the amount of effort -spent in regulating them. First, we have the completely -open board, whose sysop is off chugging brews and -watching re-runs while his users generally degenerate -over time into peevish anarchy and eventual silence. -Second comes the supervised board, where the sysop -breaks in every once in a while to tidy up, calm brawls, -issue announcements, and rid the community of dolts -and troublemakers. Third is the heavily supervised -board, which sternly urges adult and responsible behavior -and swiftly edits any message considered offensive, -impertinent, illegal or irrelevant. And last comes -the completely edited "electronic publication," which -is presented to a silent audience which is not allowed -to respond directly in any way. - -Boards can also be grouped by their degree of anonymity. -There is the completely anonymous board, where everyone -uses pseudonyms--"handles"--and even the sysop is unaware -of the user's true identity. The sysop himself is likely -pseudonymous on a board of this type. Second, and rather -more common, is the board where the sysop knows (or thinks -he knows) the true names and addresses of all users, -but the users don't know one another's names and may not know his. -Third is the board where everyone has to use real names, -and roleplaying and pseudonymous posturing are forbidden. - -Boards can be grouped by their immediacy. "Chat-lines" -are boards linking several users together over several -different phone-lines simultaneously, so that people -exchange messages at the very moment that they type. -(Many large boards feature "chat" capabilities along -with other services.) Less immediate boards, -perhaps with a single phoneline, store messages serially, -one at a time. And some boards are only open for business -in daylight hours or on weekends, which greatly slows response. -A NETWORK of boards, such as "FidoNet," can carry electronic mail -from board to board, continent to continent, across huge distances-- -but at a relative snail's pace, so that a message can take several -days to reach its target audience and elicit a reply. - -Boards can be grouped by their degree of community. -Some boards emphasize the exchange of private, -person-to-person electronic mail. Others emphasize -public postings and may even purge people who "lurk," -merely reading posts but refusing to openly participate. -Some boards are intimate and neighborly. Others are frosty -and highly technical. Some are little more than storage -dumps for software, where users "download" and "upload" programs, -but interact among themselves little if at all. - -Boards can be grouped by their ease of access. Some boards -are entirely public. Others are private and restricted only -to personal friends of the sysop. Some boards divide users by status. -On these boards, some users, especially beginners, strangers or children, -will be restricted to general topics, and perhaps forbidden to post. -Favored users, though, are granted the ability to post as they please, -and to stay "on-line" as long as they like, even to the disadvantage -of other people trying to call in. High-status users can be given access -to hidden areas in the board, such as off-color topics, private discussions, -and/or valuable software. Favored users may even become "remote sysops" -with the power to take remote control of the board through their own -home computers. Quite often "remote sysops" end up doing all the work -and taking formal control of the enterprise, despite the fact that it's -physically located in someone else's house. Sometimes several "co-sysops" -share power. - -And boards can also be grouped by size. Massive, nationwide -commercial networks, such as CompuServe, Delphi, GEnie and Prodigy, -are run on mainframe computers and are generally not considered "boards," -though they share many of their characteristics, such as electronic mail, -discussion topics, libraries of software, and persistent and growing problems -with civil-liberties issues. Some private boards have as many as -thirty phone-lines and quite sophisticated hardware. And then -there are tiny boards. - -Boards vary in popularity. Some boards are huge and crowded, -where users must claw their way in against a constant busy-signal. -Others are huge and empty--there are few things sadder than a formerly -flourishing board where no one posts any longer, and the dead conversations -of vanished users lie about gathering digital dust. Some boards are tiny -and intimate, their telephone numbers intentionally kept confidential -so that only a small number can log on. - -And some boards are UNDERGROUND. - -Boards can be mysterious entities. The activities of -their users can be hard to differentiate from conspiracy. -Sometimes they ARE conspiracies. Boards have harbored, -or have been accused of harboring, all manner of fringe groups, -and have abetted, or been accused of abetting, every manner -of frowned-upon, sleazy, radical, and criminal activity. -There are Satanist boards. Nazi boards. Pornographic boards. -Pedophile boards. Drug- dealing boards. Anarchist boards. -Communist boards. Gay and Lesbian boards (these exist in great profusion, -many of them quite lively with well-established histories). -Religious cult boards. Evangelical boards. Witchcraft -boards, hippie boards, punk boards, skateboarder boards. -Boards for UFO believers. There may well be boards for -serial killers, airline terrorists and professional assassins. -There is simply no way to tell. Boards spring up, flourish, -and disappear in large numbers, in most every corner of -the developed world. Even apparently innocuous public -boards can, and sometimes do, harbor secret areas known -only to a few. And even on the vast, public, commercial services, -private mail is very private--and quite possibly criminal. - -Boards cover most every topic imaginable and some -that are hard to imagine. They cover a vast spectrum -of social activity. However, all board users do have -something in common: their possession of computers -and phones. Naturally, computers and phones are -primary topics of conversation on almost every board. - -And hackers and phone phreaks, those utter devotees -of computers and phones, live by boards. They swarm by boards. -They are bred by boards. By the late 1980s, phone-phreak groups -and hacker groups, united by boards, had proliferated fantastically. - - -As evidence, here is a list of hacker groups compiled -by the editors of Phrack on August 8, 1988. - - -The Administration. -Advanced Telecommunications, Inc. -ALIAS. -American Tone Travelers. -Anarchy Inc. -Apple Mafia. -The Association. -Atlantic Pirates Guild. - -Bad Ass Mother Fuckers. -Bellcore. -Bell Shock Force. -Black Bag. - -Camorra. -C&M Productions. -Catholics Anonymous. -Chaos Computer Club. -Chief Executive Officers. -Circle Of Death. -Circle Of Deneb. -Club X. -Coalition of Hi-Tech -Pirates. -Coast-To-Coast. -Corrupt Computing. -Cult Of The -Dead Cow. -Custom Retaliations. - -Damage Inc. -D&B Communications. -The Danger Gang. -Dec Hunters. -Digital Gang. -DPAK. - -Eastern Alliance. -The Elite Hackers Guild. -Elite Phreakers and Hackers Club. -The Elite Society Of America. -EPG. -Executives Of Crime. -Extasyy Elite. - -Fargo 4A. -Farmers Of Doom. -The Federation. -Feds R Us. -First Class. -Five O. -Five Star. -Force Hackers. -The 414s. - -Hack-A-Trip. -Hackers Of America. -High Mountain Hackers. -High Society. -The Hitchhikers. - -IBM Syndicate. -The Ice Pirates. -Imperial Warlords. -Inner Circle. -Inner Circle II. -Insanity Inc. -International Computer Underground Bandits. - -Justice League of America. - -Kaos Inc. -Knights Of Shadow. -Knights Of The Round Table. - -League Of Adepts. -Legion Of Doom. -Legion Of Hackers. -Lords Of Chaos. -Lunatic Labs, Unlimited. - -Master Hackers. -MAD! -The Marauders. -MD/PhD. - -Metal Communications, Inc. -MetalliBashers, Inc. -MBI. - -Metro Communications. -Midwest Pirates Guild. - -NASA Elite. -The NATO Association. -Neon Knights. - -Nihilist Order. -Order Of The Rose. -OSS. - -Pacific Pirates Guild. -Phantom Access Associates. - -PHido PHreaks. -The Phirm. -Phlash. -PhoneLine Phantoms. -Phone Phreakers Of America. -Phortune 500. - -Phreak Hack Delinquents. -Phreak Hack Destroyers. - -Phreakers, Hackers, And Laundromat Employees Gang (PHALSE Gang). -Phreaks Against Geeks. -Phreaks Against Phreaks Against Geeks. -Phreaks and Hackers of America. -Phreaks Anonymous World Wide. -Project Genesis. -The Punk Mafia. - -The Racketeers. -Red Dawn Text Files. -Roscoe Gang. - - -SABRE. -Secret Circle of Pirates. -Secret Service. -707 Club. -Shadow Brotherhood. -Sharp Inc. -65C02 Elite. - -Spectral Force. -Star League. -Stowaways. -Strata-Crackers. - - -Team Hackers '86. -Team Hackers '87. - -TeleComputist Newsletter Staff. -Tribunal Of Knowledge. - -Triple Entente. -Turn Over And Die Syndrome (TOADS). - -300 Club. -1200 Club. -2300 Club. -2600 Club. -2601 Club. - -2AF. - -The United Soft WareZ Force. -United Technical Underground. - -Ware Brigade. -The Warelords. -WASP. - -Contemplating this list is an impressive, almost humbling business. -As a cultural artifact, the thing approaches poetry. - -Underground groups--subcultures--can be distinguished -from independent cultures by their habit of referring -constantly to the parent society. Undergrounds by their -nature constantly must maintain a membrane of differentiation. -Funny/distinctive clothes and hair, specialized jargon, specialized -ghettoized areas in cities, different hours of rising, working, -sleeping. . . . The digital underground, which specializes in information, -relies very heavily on language to distinguish itself. As can be seen -from this list, they make heavy use of parody and mockery. -It's revealing to see who they choose to mock. - -First, large corporations. We have the Phortune 500, -The Chief Executive Officers, Bellcore, IBM Syndicate, -SABRE (a computerized reservation service maintained -by airlines). The common use of "Inc." is telling-- -none of these groups are actual corporations, -but take clear delight in mimicking them. - -Second, governments and police. NASA Elite, NATO Association. -"Feds R Us" and "Secret Service" are fine bits of fleering boldness. -OSS--the Office of Strategic Services was the forerunner of the CIA. - -Third, criminals. Using stigmatizing pejoratives as a perverse -badge of honor is a time-honored tactic for subcultures: -punks, gangs, delinquents, mafias, pirates, bandits, racketeers. - -Specialized orthography, especially the use of "ph" for "f" -and "z" for the plural "s," are instant recognition symbols. -So is the use of the numeral "0" for the letter "O" ---computer-software orthography generally features a -slash through the zero, making the distinction obvious. - -Some terms are poetically descriptive of computer intrusion: -the Stowaways, the Hitchhikers, the PhoneLine Phantoms, Coast-to-Coast. -Others are simple bravado and vainglorious puffery. -(Note the insistent use of the terms "elite" and "master.") -Some terms are blasphemous, some obscene, others merely cryptic-- -anything to puzzle, offend, confuse, and keep the straights at bay. - -Many hacker groups further re-encrypt their names -by the use of acronyms: United Technical Underground -becomes UTU, Farmers of Doom become FoD, the United SoftWareZ -Force becomes, at its own insistence, "TuSwF," and woe to the -ignorant rodent who capitalizes the wrong letters. - -It should be further recognized that the members of these groups -are themselves pseudonymous. If you did, in fact, run across -the "PhoneLine Phantoms," you would find them to consist of -"Carrier Culprit," "The Executioner," "Black Majik," -"Egyptian Lover," "Solid State," and "Mr Icom." -"Carrier Culprit" will likely be referred to by his friends -as "CC," as in, "I got these dialups from CC of PLP." - -It's quite possible that this entire list refers to as -few as a thousand people. It is not a complete list -of underground groups--there has never been such a list, -and there never will be. Groups rise, flourish, decline, -share membership, maintain a cloud of wannabes and -casual hangers-on. People pass in and out, are ostracized, -get bored, are busted by police, or are cornered by telco -security and presented with huge bills. Many "underground -groups" are software pirates, "warez d00dz," who might break -copy protection and pirate programs, but likely wouldn't dare -to intrude on a computer-system. - -It is hard to estimate the true population of the digital -underground. There is constant turnover. Most hackers -start young, come and go, then drop out at age 22-- -the age of college graduation. And a large majority -of "hackers" access pirate boards, adopt a handle, -swipe software and perhaps abuse a phone-code or two, -while never actually joining the elite. - -Some professional informants, who make it their business -to retail knowledge of the underground to paymasters in private -corporate security, have estimated the hacker population -at as high as fifty thousand. This is likely highly inflated, -unless one counts every single teenage software pirate -and petty phone-booth thief. My best guess is about 5,000 people. -Of these, I would guess that as few as a hundred are truly "elite" ---active computer intruders, skilled enough to penetrate -sophisticated systems and truly to worry corporate security -and law enforcement. - -Another interesting speculation is whether this group -is growing or not. Young teenage hackers are often -convinced that hackers exist in vast swarms and will soon -dominate the cybernetic universe. Older and wiser -veterans, perhaps as wizened as 24 or 25 years old, -are convinced that the glory days are long gone, that the cops -have the underground's number now, and that kids these days -are dirt-stupid and just want to play Nintendo. - -My own assessment is that computer intrusion, as a non-profit act -of intellectual exploration and mastery, is in slow decline, -at least in the United States; but that electronic fraud, -especially telecommunication crime, is growing by leaps and bounds. - -One might find a useful parallel to the digital underground -in the drug underground. There was a time, now much-obscured -by historical revisionism, when Bohemians freely shared joints -at concerts, and hip, small-scale marijuana dealers might -turn people on just for the sake of enjoying a long stoned conversation -about the Doors and Allen Ginsberg. Now drugs are increasingly verboten, -except in a high-stakes, highly-criminal world of highly addictive drugs. -Over years of disenchantment and police harassment, a vaguely ideological, -free-wheeling drug underground has relinquished the business of drug-dealing -to a far more savage criminal hard-core. This is not a pleasant prospect -to contemplate, but the analogy is fairly compelling. - -What does an underground board look like? What distinguishes -it from a standard board? It isn't necessarily the conversation-- -hackers often talk about common board topics, such as hardware, software, -sex, science fiction, current events, politics, movies, personal gossip. -Underground boards can best be distinguished by their files, or "philes," -pre-composed texts which teach the techniques and ethos of the underground. -These are prized reservoirs of forbidden knowledge. Some are anonymous, -but most proudly bear the handle of the "hacker" who has created them, -and his group affiliation, if he has one. - -Here is a partial table-of-contents of philes from an underground board, -somewhere in the heart of middle America, circa 1991. The descriptions -are mostly self-explanatory. - - -BANKAMER.ZIP 5406 06-11-91 Hacking Bank America -CHHACK.ZIP 4481 06-11-91 Chilton Hacking -CITIBANK.ZIP 4118 06-11-91 Hacking Citibank -CREDIMTC.ZIP 3241 06-11-91 Hacking Mtc Credit Company -DIGEST.ZIP 5159 06-11-91 Hackers Digest -HACK.ZIP 14031 06-11-91 How To Hack -HACKBAS.ZIP 5073 06-11-91 Basics Of Hacking -HACKDICT.ZIP 42774 06-11-91 Hackers Dictionary -HACKER.ZIP 57938 06-11-91 Hacker Info -HACKERME.ZIP 3148 06-11-91 Hackers Manual -HACKHAND.ZIP 4814 06-11-91 Hackers Handbook -HACKTHES.ZIP 48290 06-11-91 Hackers Thesis -HACKVMS.ZIP 4696 06-11-91 Hacking Vms Systems -MCDON.ZIP 3830 06-11-91 Hacking Macdonalds (Home Of The Archs) -P500UNIX.ZIP 15525 06-11-91 Phortune 500 Guide To Unix -RADHACK.ZIP 8411 06-11-91 Radio Hacking -TAOTRASH.DOC 4096 12-25-89 Suggestions For Trashing -TECHHACK.ZIP 5063 06-11-91 Technical Hacking - - -The files above are do-it-yourself manuals about computer intrusion. -The above is only a small section of a much larger library of hacking -and phreaking techniques and history. We now move into a different -and perhaps surprising area. - -+------------+ - |Anarchy| -+------------+ - -ANARC.ZIP 3641 06-11-91 Anarchy Files -ANARCHST.ZIP 63703 06-11-91 Anarchist Book -ANARCHY.ZIP 2076 06-11-91 Anarchy At Home -ANARCHY3.ZIP 6982 06-11-91 Anarchy No 3 -ANARCTOY.ZIP 2361 06-11-91 Anarchy Toys -ANTIMODM.ZIP 2877 06-11-91 Anti-modem Weapons -ATOM.ZIP 4494 06-11-91 How To Make An Atom Bomb -BARBITUA.ZIP 3982 06-11-91 Barbiturate Formula -BLCKPWDR.ZIP 2810 06-11-91 Black Powder Formulas -BOMB.ZIP 3765 06-11-91 How To Make Bombs -BOOM.ZIP 2036 06-11-91 Things That Go Boom -CHLORINE.ZIP 1926 06-11-91 Chlorine Bomb -COOKBOOK.ZIP 1500 06-11-91 Anarchy Cook Book -DESTROY.ZIP 3947 06-11-91 Destroy Stuff -DUSTBOMB.ZIP 2576 06-11-91 Dust Bomb -ELECTERR.ZIP 3230 06-11-91 Electronic Terror -EXPLOS1.ZIP 2598 06-11-91 Explosives 1 -EXPLOSIV.ZIP 18051 06-11-91 More Explosives -EZSTEAL.ZIP 4521 06-11-91 Ez-stealing -FLAME.ZIP 2240 06-11-91 Flame Thrower -FLASHLT.ZIP 2533 06-11-91 Flashlight Bomb -FMBUG.ZIP 2906 06-11-91 How To Make An Fm Bug -OMEEXPL.ZIP 2139 06-11-91 Home Explosives -HOW2BRK.ZIP 3332 06-11-91 How To Break In -LETTER.ZIP 2990 06-11-91 Letter Bomb -LOCK.ZIP 2199 06-11-91 How To Pick Locks -MRSHIN.ZIP 3991 06-11-91 Briefcase Locks -NAPALM.ZIP 3563 06-11-91 Napalm At Home -NITRO.ZIP 3158 06-11-91 Fun With Nitro -PARAMIL.ZIP 2962 06-11-91 Paramilitary Info -PICKING.ZIP 3398 06-11-91 Picking Locks -PIPEBOMB.ZIP 2137 06-11-91 Pipe Bomb -POTASS.ZIP 3987 06-11-91 Formulas With Potassium -PRANK.TXT 11074 08-03-90 More Pranks To Pull On Idiots! -REVENGE.ZIP 4447 06-11-91 Revenge Tactics -ROCKET.ZIP 2590 06-11-91 Rockets For Fun -SMUGGLE.ZIP 3385 06-11-91 How To Smuggle - -HOLY COW! The damned thing is full of stuff about bombs! - -What are we to make of this? - -First, it should be acknowledged that spreading -knowledge about demolitions to teenagers is a highly and -deliberately antisocial act. It is not, however, illegal. - -Second, it should be recognized that most of these -philes were in fact WRITTEN by teenagers. Most adult -American males who can remember their teenage years -will recognize that the notion of building a flamethrower -in your garage is an incredibly neat-o idea. ACTUALLY, -building a flamethrower in your garage, however, is -fraught with discouraging difficulty. Stuffing gunpowder -into a booby-trapped flashlight, so as to blow the arm off -your high-school vice-principal, can be a thing of dark -beauty to contemplate. Actually committing assault by -explosives will earn you the sustained attention of the -federal Bureau of Alcohol, Tobacco and Firearms. - -Some people, however, will actually try these plans. -A determinedly murderous American teenager can probably -buy or steal a handgun far more easily than he can brew -fake "napalm" in the kitchen sink. Nevertheless, -if temptation is spread before people, a certain number -will succumb, and a small minority will actually attempt -these stunts. A large minority of that small minority -will either fail or, quite likely, maim themselves, -since these "philes" have not been checked for accuracy, -are not the product of professional experience, -and are often highly fanciful. But the gloating menace -of these philes is not to be entirely dismissed. - -Hackers may not be "serious" about bombing; if they were, -we would hear far more about exploding flashlights, homemade bazookas, -and gym teachers poisoned by chlorine and potassium. -However, hackers are VERY serious about forbidden knowledge. -They are possessed not merely by curiosity, but by -a positive LUST TO KNOW. The desire to know what -others don't is scarcely new. But the INTENSITY -of this desire, as manifested by these young technophilic -denizens of the Information Age, may in fact BE new, -and may represent some basic shift in social values-- -a harbinger of what the world may come to, as society -lays more and more value on the possession, -assimilation and retailing of INFORMATION -as a basic commodity of daily life. - -There have always been young men with obsessive interests -in these topics. Never before, however, have they been able -to network so extensively and easily, and to propagandize -their interests with impunity to random passers-by. -High-school teachers will recognize that there's always -one in a crowd, but when the one in a crowd escapes control -by jumping into the phone-lines, and becomes a hundred such kids -all together on a board, then trouble is brewing visibly. -The urge of authority to DO SOMETHING, even something drastic, -is hard to resist. And in 1990, authority did something. -In fact authority did a great deal. - -# - -The process by which boards create hackers goes something -like this. A youngster becomes interested in computers-- -usually, computer games. He hears from friends that -"bulletin boards" exist where games can be obtained for free. -(Many computer games are "freeware," not copyrighted-- -invented simply for the love of it and given away to the public; -some of these games are quite good.) He bugs his parents for a modem, -or quite often, uses his parents' modem. - -The world of boards suddenly opens up. Computer games -can be quite expensive, real budget-breakers for a kid, -but pirated games, stripped of copy protection, are cheap or free. -They are also illegal, but it is very rare, almost unheard of, -for a small-scale software pirate to be prosecuted. -Once "cracked" of its copy protection, the program, -being digital data, becomes infinitely reproducible. -Even the instructions to the game, any manuals that accompany it, -can be reproduced as text files, or photocopied from legitimate sets. -Other users on boards can give many useful hints in game-playing tactics. -And a youngster with an infinite supply of free computer games can -certainly cut quite a swath among his modem-less friends. - -And boards are pseudonymous. No one need know that you're -fourteen years old--with a little practice at subterfuge, -you can talk to adults about adult things, and be accepted -and taken seriously! You can even pretend to be a girl, -or an old man, or anybody you can imagine. If you find this -kind of deception gratifying, there is ample opportunity -to hone your ability on boards. - -But local boards can grow stale. And almost every board maintains -a list of phone-numbers to other boards, some in distant, tempting, -exotic locales. Who knows what they're up to, in Oregon or Alaska -or Florida or California? It's very easy to find out--just order -the modem to call through its software--nothing to this, just typing -on a keyboard, the same thing you would do for most any computer game. -The machine reacts swiftly and in a few seconds you are talking to -a bunch of interesting people on another seaboard. - -And yet the BILLS for this trivial action can be staggering! -Just by going tippety-tap with your fingers, you may have -saddled your parents with four hundred bucks in long-distance charges, -and gotten chewed out but good. That hardly seems fair. - -How horrifying to have made friends in another state -and to be deprived of their company--and their software-- -just because telephone companies demand absurd amounts of money! -How painful, to be restricted to boards in one's own AREA CODE-- -what the heck is an "area code" anyway, and what makes it so special? -A few grumbles, complaints, and innocent questions of this sort -will often elicit a sympathetic reply from another board user-- -someone with some stolen codes to hand. You dither a while, -knowing this isn't quite right, then you make up your mind -to try them anyhow--AND THEY WORK! Suddenly you're doing something -even your parents can't do. Six months ago you were just some kid--now, -you're the Crimson Flash of Area Code 512! You're bad--you're nationwide! - -Maybe you'll stop at a few abused codes. Maybe you'll decide that -boards aren't all that interesting after all, that it's wrong, -not worth the risk --but maybe you won't. The next step -is to pick up your own repeat-dialling program-- -to learn to generate your own stolen codes. -(This was dead easy five years ago, much harder -to get away with nowadays, but not yet impossible.) -And these dialling programs are not complex or intimidating-- -some are as small as twenty lines of software. - -Now, you too can share codes. You can trade codes to learn -other techniques. If you're smart enough to catch on, -and obsessive enough to want to bother, and ruthless enough -to start seriously bending rules, then you'll get better, fast. -You start to develop a rep. You move up to a heavier class -of board--a board with a bad attitude, the kind of board -that naive dopes like your classmates and your former self -have never even heard of! You pick up the jargon of phreaking -and hacking from the board. You read a few of those anarchy philes-- -and man, you never realized you could be a real OUTLAW without -ever leaving your bedroom. - -You still play other computer games, but now you have a new -and bigger game. This one will bring you a different kind of status -than destroying even eight zillion lousy space invaders. - -Hacking is perceived by hackers as a "game." This is -not an entirely unreasonable or sociopathic perception. -You can win or lose at hacking, succeed or fail, -but it never feels "real." It's not simply that -imaginative youngsters sometimes have a hard time -telling "make-believe" from "real life." Cyberspace -is NOT REAL! "Real" things are physical objects -like trees and shoes and cars. Hacking takes place -on a screen. Words aren't physical, numbers -(even telephone numbers and credit card numbers) -aren't physical. Sticks and stones may break my bones, -but data will never hurt me. Computers SIMULATE reality, -like computer games that simulate tank battles or dogfights -or spaceships. Simulations are just make-believe, -and the stuff in computers is NOT REAL. - -Consider this: if "hacking" is supposed to be so serious and -real-life and dangerous, then how come NINE-YEAR-OLD KIDS have -computers and modems? You wouldn't give a nine year old his own car, -or his own rifle, or his own chainsaw--those things are "real." - -People underground are perfectly aware that the "game" -is frowned upon by the powers that be. Word gets around -about busts in the underground. Publicizing busts is one -of the primary functions of pirate boards, but they also -promulgate an attitude about them, and their own idiosyncratic -ideas of justice. The users of underground boards won't complain -if some guy is busted for crashing systems, spreading viruses, -or stealing money by wire-fraud. They may shake their heads -with a sneaky grin, but they won't openly defend these practices. -But when a kid is charged with some theoretical amount of theft: -$233,846.14, for instance, because he sneaked into a computer -and copied something, and kept it in his house on a floppy disk-- -this is regarded as a sign of near-insanity from prosecutors, -a sign that they've drastically mistaken the immaterial game -of computing for their real and boring everyday world -of fatcat corporate money. - -It's as if big companies and their suck-up lawyers -think that computing belongs to them, and they can -retail it with price stickers, as if it were boxes -of laundry soap! But pricing "information" is like -trying to price air or price dreams. Well, anybody -on a pirate board knows that computing can be, -and ought to be, FREE. Pirate boards are little -independent worlds in cyberspace, and they don't belong -to anybody but the underground. Underground boards -aren't "brought to you by Procter & Gamble." - -To log on to an underground board can mean to -experience liberation, to enter a world where, -for once, money isn't everything and adults -don't have all the answers. - -Let's sample another vivid hacker manifesto. Here are -some excerpts from "The Conscience of a Hacker," by "The Mentor," -from Phrack Volume One, Issue 7, Phile 3. - -"I made a discovery today. I found a computer. -Wait a second, this is cool. It does what I want it to. -If it makes a mistake, it's because I screwed it up. -Not because it doesn't like me. (. . .) -"And then it happened. . .a door opened to a world. . . -rushing through the phone line like heroin through an -addict's veins, an electronic pulse is sent out, -a refuge from day-to-day incompetencies is sought. . . -a board is found. `This is it. . .this is where I belong. . .' -"I know everyone here. . .even if I've never met them, -never talked to them, may never hear from them again. . . -I know you all. . . (. . .) - -"This is our world now. . .the world of the electron -and the switch, the beauty of the baud. We make use of a -service already existing without paying for what could be -dirt-cheap if it wasn't run by profiteering gluttons, and you -call us criminals. We explore. . .and you call us criminals. -We seek after knowledge. . .and you call us criminals. -We exist without skin color, without nationality, -without religious bias. . .and you call us criminals. -You build atomic bombs, you wage wars, you murder, -cheat and lie to us and try to make us believe that -it's for our own good, yet we're the criminals. - -"Yes, I am a criminal. My crime is that of curiosity. -My crime is that of judging people by what they say and think, -not what they look like. My crime is that of outsmarting you, -something that you will never forgive me for." - -# - -There have been underground boards almost as long -as there have been boards. One of the first was 8BBS, -which became a stronghold of the West Coast phone-phreak elite. -After going on-line in March 1980, 8BBS sponsored "Susan Thunder," -and "Tuc," and, most notoriously, "the Condor." "The Condor" -bore the singular distinction of becoming the most vilified -American phreak and hacker ever. Angry underground associates, -fed up with Condor's peevish behavior, turned him in to police, -along with a heaping double-helping of outrageous hacker legendry. -As a result, Condor was kept in solitary confinement for seven months, -for fear that he might start World War Three by triggering missile silos -from the prison payphone. (Having served his time, Condor is now -walking around loose; WWIII has thus far conspicuously failed to occur.) - -The sysop of 8BBS was an ardent free-speech enthusiast -who simply felt that ANY attempt to restrict the expression -of his users was unconstitutional and immoral. -Swarms of the technically curious entered 8BBS -and emerged as phreaks and hackers, until, in 1982, -a friendly 8BBS alumnus passed the sysop a new modem -which had been purchased by credit-card fraud. -Police took this opportunity to seize the entire board -and remove what they considered an attractive nuisance. - -Plovernet was a powerful East Coast pirate board -that operated in both New York and Florida. -Owned and operated by teenage hacker "Quasi Moto," -Plovernet attracted five hundred eager users in 1983. -"Emmanuel Goldstein" was one-time co-sysop of Plovernet, -along with "Lex Luthor," founder of the "Legion of Doom" group. -Plovernet bore the signal honor of being the original home -of the "Legion of Doom," about which the reader will be hearing -a great deal, soon. - -"Pirate-80," or "P-80," run by a sysop known as "Scan-Man," -got into the game very early in Charleston, and continued -steadily for years. P-80 flourished so flagrantly that -even its most hardened users became nervous, and some -slanderously speculated that "Scan Man" must have ties -to corporate security, a charge he vigorously denied. - -"414 Private" was the home board for the first GROUP -to attract conspicuous trouble, the teenage "414 Gang," -whose intrusions into Sloan-Kettering Cancer Center and -Los Alamos military computers were to be a nine-days-wonder in 1982. - -At about this time, the first software piracy boards -began to open up, trading cracked games for the Atari 800 -and the Commodore C64. Naturally these boards were -heavily frequented by teenagers. And with the 1983 -release of the hacker-thriller movie War Games, -the scene exploded. It seemed that every kid -in America had demanded and gotten a modem for Christmas. -Most of these dabbler wannabes put their modems in the attic -after a few weeks, and most of the remainder minded their -P's and Q's and stayed well out of hot water. But some -stubborn and talented diehards had this hacker kid in -War Games figured for a happening dude. They simply -could not rest until they had contacted the underground-- -or, failing that, created their own. - -In the mid-80s, underground boards sprang up like digital fungi. -ShadowSpawn Elite. Sherwood Forest I, II, and III. -Digital Logic Data Service in Florida, sysoped by no less -a man than "Digital Logic" himself; Lex Luthor of the -Legion of Doom was prominent on this board, since it -was in his area code. Lex's own board, "Legion of Doom," -started in 1984. The Neon Knights ran a network of Apple- -hacker boards: Neon Knights North, South, East and West. -Free World II was run by "Major Havoc." Lunatic Labs -is still in operation as of this writing. Dr. Ripco -in Chicago, an anything-goes anarchist board with an -extensive and raucous history, was seized by Secret Service -agents in 1990 on Sundevil day, but up again almost immediately, -with new machines and scarcely diminished vigor. - -The St. Louis scene was not to rank with major centers -of American hacking such as New York and L.A. But St. -Louis did rejoice in possession of "Knight Lightning" -and "Taran King," two of the foremost JOURNALISTS native -to the underground. Missouri boards like Metal Shop, -Metal Shop Private, Metal Shop Brewery, may not have -been the heaviest boards around in terms of illicit -expertise. But they became boards where hackers could -exchange social gossip and try to figure out what the -heck was going on nationally--and internationally. -Gossip from Metal Shop was put into the form of news files, -then assembled into a general electronic publication, -Phrack, a portmanteau title coined from "phreak" and "hack." -The Phrack editors were as obsessively curious about other -hackers as hackers were about machines. - -Phrack, being free of charge and lively reading, began -to circulate throughout the underground. As Taran King -and Knight Lightning left high school for college, -Phrack began to appear on mainframe machines linked to BITNET, -and, through BITNET to the "Internet," that loose but -extremely potent not-for-profit network where academic, -governmental and corporate machines trade data through -the UNIX TCP/IP protocol. (The "Internet Worm" of -November 2-3,1988, created by Cornell grad student Robert Morris, -was to be the largest and best-publicized computer-intrusion scandal -to date. Morris claimed that his ingenious "worm" program was meant -to harmlessly explore the Internet, but due to bad programming, -the Worm replicated out of control and crashed some six thousand -Internet computers. Smaller-scale and less ambitious Internet hacking -was a standard for the underground elite.) - -Most any underground board not hopelessly lame and out-of-it -would feature a complete run of Phrack--and, possibly, -the lesser-known standards of the underground: -the Legion of Doom Technical Journal, the obscene -and raucous Cult of the Dead Cow files, P/HUN magazine, -Pirate, the Syndicate Reports, and perhaps the highly -anarcho-political Activist Times Incorporated. - -Possession of Phrack on one's board was prima facie -evidence of a bad attitude. Phrack was seemingly everywhere, -aiding, abetting, and spreading the underground ethos. -And this did not escape the attention of corporate security -or the police. - -We now come to the touchy subject of police and boards. -Police, do, in fact, own boards. In 1989, there were -police-sponsored boards in California, Colorado, Florida, -Georgia, Idaho, Michigan, Missouri, Texas, and Virginia: -boards such as "Crime Bytes," "Crimestoppers," "All Points" -and "Bullet-N-Board." Police officers, as private computer -enthusiasts, ran their own boards in Arizona, California, -Colorado, Connecticut, Florida, Missouri, Maryland, -New Mexico, North Carolina, Ohio, Tennessee and Texas. -Police boards have often proved helpful in community relations. -Sometimes crimes are reported on police boards. - -Sometimes crimes are COMMITTED on police boards. -This has sometimes happened by accident, as naive hackers -blunder onto police boards and blithely begin offering telephone codes. -Far more often, however, it occurs through the now almost-traditional -use of "sting boards." The first police sting-boards were established -in 1985: "Underground Tunnel" in Austin, Texas, whose sysop -Sgt. Robert Ansley called himself "Pluto"--"The Phone Company" -in Phoenix, Arizona, run by Ken MacLeod of the Maricopa County -Sheriff's office--and Sgt. Dan Pasquale's board in Fremont, California. -Sysops posed as hackers, and swiftly garnered coteries of ardent users, -who posted codes and loaded pirate software with abandon, -and came to a sticky end. - -Sting boards, like other boards, are cheap to operate, -very cheap by the standards of undercover police operations. -Once accepted by the local underground, sysops will likely be -invited into other pirate boards, where they can compile more dossiers. -And when the sting is announced and the worst offenders arrested, -the publicity is generally gratifying. The resultant paranoia -in the underground--perhaps more justly described as a "deterrence effect"-- -tends to quell local lawbreaking for quite a while. - -Obviously police do not have to beat the underbrush for hackers. -On the contrary, they can go trolling for them. Those caught -can be grilled. Some become useful informants. They can lead -the way to pirate boards all across the country. - -And boards all across the country showed the sticky -fingerprints of Phrack, and of that loudest and most -flagrant of all underground groups, the "Legion of Doom." - -The term "Legion of Doom" came from comic books. The Legion of Doom, -a conspiracy of costumed super- villains headed by the chrome-domed -criminal ultra- mastermind Lex Luthor, gave Superman a lot of four-color -graphic trouble for a number of decades. Of course, Superman, -that exemplar of Truth, Justice, and the American Way, -always won in the long run. This didn't matter to the hacker Doomsters-- -"Legion of Doom" was not some thunderous and evil Satanic reference, -it was not meant to be taken seriously. "Legion of Doom" came -from funny-books and was supposed to be funny. - -"Legion of Doom" did have a good mouthfilling ring to it, though. -It sounded really cool. Other groups, such as the "Farmers of Doom," -closely allied to LoD, recognized this grandiloquent quality, -and made fun of it. There was even a hacker group called -"Justice League of America," named after Superman's club -of true-blue crimefighting superheros. - -But they didn't last; the Legion did. - -The original Legion of Doom, hanging out on Quasi Moto's Plovernet board, -were phone phreaks. They weren't much into computers. "Lex Luthor" himself -(who was under eighteen when he formed the Legion) was a COSMOS expert, -COSMOS being the "Central System for Mainframe Operations," -a telco internal computer network. Lex would eventually become -quite a dab hand at breaking into IBM mainframes, but although -everyone liked Lex and admired his attitude, he was not considered -a truly accomplished computer intruder. Nor was he the "mastermind" -of the Legion of Doom--LoD were never big on formal leadership. -As a regular on Plovernet and sysop of his "Legion of Doom BBS," -Lex was the Legion's cheerleader and recruiting officer. - -Legion of Doom began on the ruins of an earlier phreak group, -The Knights of Shadow. Later, LoD was to subsume the personnel -of the hacker group "Tribunal of Knowledge." People came and went -constantly in LoD; groups split up or formed offshoots. - -Early on, the LoD phreaks befriended a few computer-intrusion -enthusiasts, who became the associated "Legion of Hackers." -Then the two groups conflated into the "Legion of Doom/Hackers," -or LoD/H. When the original "hacker" wing, Messrs. "Compu-Phreak" -and "Phucked Agent 04," found other matters to occupy their time, -the extra "/H" slowly atrophied out of the name; but by this time -the phreak wing, Messrs. Lex Luthor, "Blue Archer," "Gary Seven," -"Kerrang Khan," "Master of Impact," "Silver Spy," "The Marauder," -and "The Videosmith," had picked up a plethora of intrusion -expertise and had become a force to be reckoned with. - -LoD members seemed to have an instinctive understanding -that the way to real power in the underground lay through -covert publicity. LoD were flagrant. Not only was it one -of the earliest groups, but the members took pains to widely -distribute their illicit knowledge. Some LoD members, -like "The Mentor," were close to evangelical about it. -Legion of Doom Technical Journal began to show up on boards -throughout the underground. - -LoD Technical Journal was named in cruel parody -of the ancient and honored AT&T Technical Journal. -The material in these two publications was quite similar-- -much of it, adopted from public journals and discussions -in the telco community. And yet, the predatory attitude -of LoD made even its most innocuous data seem deeply sinister; -an outrage; a clear and present danger. - -To see why this should be, let's consider the following -(invented) paragraphs, as a kind of thought experiment. - -(A) "W. Fred Brown, AT&T Vice President for -Advanced Technical Development, testified May 8 -at a Washington hearing of the National Telecommunications -and Information Administration (NTIA), regarding -Bellcore's GARDEN project. GARDEN (Generalized -Automatic Remote Distributed Electronic Network) is a -telephone-switch programming tool that makes it possible -to develop new telecom services, including hold-on-hold -and customized message transfers, from any keypad terminal, -within seconds. The GARDEN prototype combines centrex -lines with a minicomputer using UNIX operating system software." - -(B) "Crimson Flash 512 of the Centrex Mobsters reports: -D00dz, you wouldn't believe this GARDEN bullshit Bellcore's -just come up with! Now you don't even need a lousy Commodore -to reprogram a switch--just log on to GARDEN as a technician, -and you can reprogram switches right off the keypad in any -public phone booth! You can give yourself hold-on-hold -and customized message transfers, and best of all, -the thing is run off (notoriously insecure) centrex lines -using--get this--standard UNIX software! Ha ha ha ha!" - -Message (A), couched in typical techno-bureaucratese, -appears tedious and almost unreadable. (A) scarcely seems -threatening or menacing. Message (B), on the other hand, -is a dreadful thing, prima facie evidence of a dire conspiracy, -definitely not the kind of thing you want your teenager reading. - -The INFORMATION, however, is identical. It is PUBLIC -information, presented before the federal government in -an open hearing. It is not "secret." It is not "proprietary." -It is not even "confidential." On the contrary, the -development of advanced software systems is a matter -of great public pride to Bellcore. - -However, when Bellcore publicly announces a project of this kind, -it expects a certain attitude from the public--something along -the lines of GOSH WOW, YOU GUYS ARE GREAT, KEEP THAT UP, WHATEVER IT IS-- -certainly not cruel mimickry, one-upmanship and outrageous speculations -about possible security holes. - -Now put yourself in the place of a policeman confronted by -an outraged parent, or telco official, with a copy of Version (B). -This well-meaning citizen, to his horror, has discovered -a local bulletin-board carrying outrageous stuff like (B), -which his son is examining with a deep and unhealthy interest. -If (B) were printed in a book or magazine, you, as an American -law enforcement officer, would know that it would take -a hell of a lot of trouble to do anything about it; -but it doesn't take technical genius to recognize that -if there's a computer in your area harboring stuff like (B), -there's going to be trouble. - -In fact, if you ask around, any computer-literate cop -will tell you straight out that boards with stuff like (B) -are the SOURCE of trouble. And the WORST source of trouble -on boards are the ringleaders inventing and spreading stuff like (B). -If it weren't for these jokers, there wouldn't BE any trouble. - -And Legion of Doom were on boards like nobody else. -Plovernet. The Legion of Doom Board. The Farmers of Doom Board. -Metal Shop. OSUNY. Blottoland. Private Sector. Atlantis. -Digital Logic. Hell Phrozen Over. - -LoD members also ran their own boards. "Silver Spy" started -his own board, "Catch-22," considered one of the heaviest around. -So did "Mentor," with his "Phoenix Project." When they didn't run boards -themselves, they showed up on other people's boards, to brag, boast, -and strut. And where they themselves didn't go, their philes went, -carrying evil knowledge and an even more evil attitude. - -As early as 1986, the police were under the vague impression -that EVERYONE in the underground was Legion of Doom. -LoD was never that large--considerably smaller than either -"Metal Communications" or "The Administration," for instance-- -but LoD got tremendous press. Especially in Phrack, -which at times read like an LoD fan magazine; and Phrack -was everywhere, especially in the offices of telco security. -You couldn't GET busted as a phone phreak, a hacker, -or even a lousy codes kid or warez dood, without the cops -asking if you were LoD. - -This was a difficult charge to deny, as LoD never -distributed membership badges or laminated ID cards. -If they had, they would likely have died out quickly, -for turnover in their membership was considerable. -LoD was less a high-tech street-gang than an ongoing -state-of-mind. LoD was the Gang That Refused to Die. -By 1990, LoD had RULED for ten years, and it seemed WEIRD -to police that they were continually busting people who were -only sixteen years old. All these teenage small-timers -were pleading the tiresome hacker litany of "just curious, -no criminal intent." Somewhere at the center of this -conspiracy there had to be some serious adult masterminds, -not this seemingly endless supply of myopic suburban -white kids with high SATs and funny haircuts. - -There was no question that most any American hacker -arrested would "know" LoD. They knew the handles -of contributors to LoD Tech Journal, and were likely -to have learned their craft through LoD boards and LoD activism. -But they'd never met anyone from LoD. Even some of the -rotating cadre who were actually and formally "in LoD" -knew one another only by board-mail and pseudonyms. -This was a highly unconventional profile for a criminal conspiracy. -Computer networking, and the rapid evolution of the digital underground, -made the situation very diffuse and confusing. - -Furthermore, a big reputation in the digital underground -did not coincide with one's willingness to commit "crimes." -Instead, reputation was based on cleverness and technical mastery. -As a result, it often seemed that the HEAVIER the hackers were, -the LESS likely they were to have committed any kind of common, -easily prosecutable crime. There were some hackers who could really steal. -And there were hackers who could really hack. But the two groups didn't seem -to overlap much, if at all. For instance, most people in the underground -looked up to "Emmanuel Goldstein" of 2600 as a hacker demigod. -But Goldstein's publishing activities were entirely legal-- -Goldstein just printed dodgy stuff and talked about politics, -he didn't even hack. When you came right down to it, -Goldstein spent half his time complaining that computer security -WASN'T STRONG ENOUGH and ought to be drastically improved -across the board! - -Truly heavy-duty hackers, those with serious technical skills -who had earned the respect of the underground, never stole money -or abused credit cards. Sometimes they might abuse phone-codes-- -but often, they seemed to get all the free phone-time they wanted -without leaving a trace of any kind. - -The best hackers, the most powerful and technically accomplished, -were not professional fraudsters. They raided computers habitually, -but wouldn't alter anything, or damage anything. They didn't even steal -computer equipment--most had day-jobs messing with hardware, -and could get all the cheap secondhand equipment they wanted. -The hottest hackers, unlike the teenage wannabes, weren't snobs -about fancy or expensive hardware. Their machines tended to be -raw second-hand digital hot-rods full of custom add-ons that -they'd cobbled together out of chickenwire, memory chips and spit. -Some were adults, computer software writers and consultants by trade, -and making quite good livings at it. Some of them ACTUALLY WORKED -FOR THE PHONE COMPANY--and for those, the "hackers" actually found -under the skirts of Ma Bell, there would be little mercy in 1990. - -It has long been an article of faith in the -underground that the "best" hackers never get caught. -They're far too smart, supposedly. They never get caught -because they never boast, brag, or strut. These demigods -may read underground boards (with a condescending smile), -but they never say anything there. The "best" hackers, -according to legend, are adult computer professionals, -such as mainframe system administrators, who already know -the ins and outs of their particular brand of security. -Even the "best" hacker can't break in to just any computer at random: -the knowledge of security holes is too specialized, varying widely -with different software and hardware. But if people are employed to run, -say, a UNIX mainframe or a VAX/VMS machine, then they tend to learn -security from the inside out. Armed with this knowledge, -they can look into most anybody else's UNIX or VMS -without much trouble or risk, if they want to. -And, according to hacker legend, of course they want to, -so of course they do. They just don't make a big deal -of what they've done. So nobody ever finds out. - -It is also an article of faith in the underground that -professional telco people "phreak" like crazed weasels. -OF COURSE they spy on Madonna's phone calls--I mean, -WOULDN'T YOU? Of course they give themselves free long- -distance--why the hell should THEY pay, they're running -the whole shebang! - -It has, as a third matter, long been an article of faith -that any hacker caught can escape serious punishment if -he confesses HOW HE DID IT. Hackers seem to believe -that governmental agencies and large corporations are -blundering about in cyberspace like eyeless jellyfish -or cave salamanders. They feel that these large -but pathetically stupid organizations will proffer up -genuine gratitude, and perhaps even a security post -and a big salary, to the hot-shot intruder who will deign -to reveal to them the supreme genius of his modus operandi. - -In the case of longtime LoD member "Control-C," -this actually happened, more or less. Control-C had led -Michigan Bell a merry chase, and when captured in 1987, -he turned out to be a bright and apparently physically -harmless young fanatic, fascinated by phones. There was -no chance in hell that Control-C would actually repay the -enormous and largely theoretical sums in long-distance -service that he had accumulated from Michigan Bell. -He could always be indicted for fraud or computer-intrusion, -but there seemed little real point in this--he hadn't -physically damaged any computer. He'd just plead guilty, -and he'd likely get the usual slap-on-the-wrist, -and in the meantime it would be a big hassle for Michigan Bell -just to bring up the case. But if kept on the payroll, -he might at least keep his fellow hackers at bay. - -There were uses for him. For instance, a contrite -Control-C was featured on Michigan Bell internal posters, -sternly warning employees to shred their trash. -He'd always gotten most of his best inside info from -"trashing"--raiding telco dumpsters, for useful data -indiscreetly thrown away. He signed these posters, too. -Control-C had become something like a Michigan Bell mascot. -And in fact, Control-C DID keep other hackers at bay. -Little hackers were quite scared of Control-C and his -heavy-duty Legion of Doom friends. And big hackers WERE -his friends and didn't want to screw up his cushy situation. - -No matter what one might say of LoD, they did stick together. -When "Wasp," an apparently genuinely malicious New York hacker, -began crashing Bellcore machines, Control-C received swift volunteer -help from "the Mentor" and the Georgia LoD wing made up of -"The Prophet," "Urvile," and "Leftist." Using Mentor's Phoenix -Project board to coordinate, the Doomsters helped telco security -to trap Wasp, by luring him into a machine with a tap -and line-trace installed. Wasp lost. LoD won! And my, did they brag. - -Urvile, Prophet and Leftist were well-qualified for this activity, -probably more so even than the quite accomplished Control-C. -The Georgia boys knew all about phone switching-stations. -Though relative johnny-come-latelies in the Legion of Doom, -they were considered some of LoD's heaviest guys, -into the hairiest systems around. They had the good fortune -to live in or near Atlanta, home of the sleepy and apparently -tolerant BellSouth RBOC. - -As RBOC security went, BellSouth were "cake." US West (of Arizona, -the Rockies and the Pacific Northwest) were tough and aggressive, -probably the heaviest RBOC around. Pacific Bell, California's PacBell, -were sleek, high-tech, and longtime veterans of the LA phone-phreak wars. -NYNEX had the misfortune to run the New York City area, and were warily -prepared for most anything. Even Michigan Bell, a division of the -Ameritech RBOC, at least had the elementary sense to hire their own hacker -as a useful scarecrow. But BellSouth, even though their corporate P.R. -proclaimed them to have "Everything You Expect From a Leader," were pathetic. - -When rumor about LoD's mastery of Georgia's switching network got around -to BellSouth through Bellcore and telco security scuttlebutt, -they at first refused to believe it. If you paid serious attention -to every rumor out and about these hacker kids, you would hear all kinds -of wacko saucer-nut nonsense: that the National Security Agency -monitored all American phone calls, that the CIA and DEA tracked -traffic on bulletin-boards with word-analysis programs, -that the Condor could start World War III from a payphone. - -If there were hackers into BellSouth switching-stations, then how come -nothing had happened? Nothing had been hurt. BellSouth's machines -weren't crashing. BellSouth wasn't suffering especially badly from fraud. -BellSouth's customers weren't complaining. BellSouth was headquartered -in Atlanta, ambitious metropolis of the new high-tech Sunbelt; -and BellSouth was upgrading its network by leaps and bounds, -digitizing the works left right and center. They could hardly be -considered sluggish or naive. BellSouth's technical expertise -was second to none, thank you kindly. But then came the Florida business. - -On June 13, 1989, callers to the Palm Beach County Probation Department, -in Delray Beach, Florida, found themselves involved in a remarkable -discussion with a phone-sex worker named "Tina" in New York State. -Somehow, ANY call to this probation office near Miami was instantly -and magically transported across state lines, at no extra charge to the user, -to a pornographic phone-sex hotline hundreds of miles away! - -This practical joke may seem utterly hilarious at first hearing, -and indeed there was a good deal of chuckling about it in -phone phreak circles, including the Autumn 1989 issue of 2600. -But for Southern Bell (the division of the BellSouth RBOC -supplying local service for Florida, Georgia, North Carolina -and South Carolina), this was a smoking gun. For the first time ever, -a computer intruder had broken into a BellSouth central office -switching station and re-programmed it! - -Or so BellSouth thought in June 1989. Actually, LoD members had been -frolicking harmlessly in BellSouth switches since September 1987. -The stunt of June 13--call-forwarding a number through manipulation -of a switching station--was child's play for hackers as accomplished -as the Georgia wing of LoD. Switching calls interstate sounded like -a big deal, but it took only four lines of code to accomplish this. -An easy, yet more discreet, stunt, would be to call-forward another -number to your own house. If you were careful and considerate, -and changed the software back later, then not a soul would know. -Except you. And whoever you had bragged to about it. - -As for BellSouth, what they didn't know wouldn't hurt them. - -Except now somebody had blown the whole thing wide open, and BellSouth knew. - -A now alerted and considerably paranoid BellSouth began searching switches -right and left for signs of impropriety, in that hot summer of 1989. -No fewer than forty-two BellSouth employees were put on 12-hour shifts, -twenty-four hours a day, for two solid months, poring over records -and monitoring computers for any sign of phony access. These forty-two -overworked experts were known as BellSouth's "Intrusion Task Force." - -What the investigators found astounded them. Proprietary telco databases -had been manipulated: phone numbers had been created out of thin air, -with no users' names and no addresses. And perhaps worst of all, -no charges and no records of use. The new digital ReMOB (Remote Observation) -diagnostic feature had been extensively tampered with--hackers had learned to -reprogram ReMOB software, so that they could listen in on any switch-routed -call at their leisure! They were using telco property to SPY! - -The electrifying news went out throughout law enforcement in 1989. -It had never really occurred to anyone at BellSouth that their prized -and brand-new digital switching-stations could be RE-PROGRAMMED. -People seemed utterly amazed that anyone could have the nerve. -Of course these switching stations were "computers," and everybody -knew hackers liked to "break into computers:" but telephone people's -computers were DIFFERENT from normal people's computers. - -The exact reason WHY these computers were "different" was -rather ill-defined. It certainly wasn't the extent of their security. -The security on these BellSouth computers was lousy; the AIMSX computers, -for instance, didn't even have passwords. But there was no question that -BellSouth strongly FELT that their computers were very different indeed. -And if there were some criminals out there who had not gotten that message, -BellSouth was determined to see that message taught. - -After all, a 5ESS switching station was no mere bookkeeping system for -some local chain of florists. Public service depended on these stations. -Public SAFETY depended on these stations. - -And hackers, lurking in there call-forwarding or ReMobbing, could spy -on anybody in the local area! They could spy on telco officials! -They could spy on police stations! They could spy on local offices -of the Secret Service. . . . - -In 1989, electronic cops and hacker-trackers began using scrambler-phones -and secured lines. It only made sense. There was no telling who was into -those systems. Whoever they were, they sounded scary. This was some -new level of antisocial daring. Could be West German hackers, in the pay -of the KGB. That too had seemed a weird and farfetched notion, -until Clifford Stoll had poked and prodded a sluggish Washington -law-enforcement bureaucracy into investigating a computer intrusion -that turned out to be exactly that--HACKERS, IN THE PAY OF THE KGB! -Stoll, the systems manager for an Internet lab in Berkeley California, -had ended up on the front page of the New Nork Times, proclaimed a national -hero in the first true story of international computer espionage. -Stoll's counterspy efforts, which he related in a bestselling book, -The Cuckoo's Egg, in 1989, had established the credibility of `hacking' -as a possible threat to national security. The United States Secret Service -doesn't mess around when it suspects a possible action by a foreign -intelligence apparat. - -The Secret Service scrambler-phones and secured lines put -a tremendous kink in law enforcement's ability to operate freely; -to get the word out, cooperate, prevent misunderstandings. -Nevertheless, 1989 scarcely seemed the time for half-measures. -If the police and Secret Service themselves were not operationally secure, -then how could they reasonably demand measures of security from -private enterprise? At least, the inconvenience made people aware -of the seriousness of the threat. - -If there was a final spur needed to get the police off the dime, -it came in the realization that the emergency 911 system was vulnerable. -The 911 system has its own specialized software, but it is run on the same -digital switching systems as the rest of the telephone network. -911 is not physically different from normal telephony. But it is -certainly culturally different, because this is the area of -telephonic cyberspace reserved for the police and emergency services. - -Your average policeman may not know much about hackers or phone-phreaks. -Computer people are weird; even computer COPS are rather weird; -the stuff they do is hard to figure out. But a threat to the 911 system -is anything but an abstract threat. If the 911 system goes, people can die. - -Imagine being in a car-wreck, staggering to a phone-booth, -punching 911 and hearing "Tina" pick up the phone-sex line -somewhere in New York! The situation's no longer comical, somehow. - -And was it possible? No question. Hackers had attacked 911 -systems before. Phreaks can max-out 911 systems just by siccing -a bunch of computer-modems on them in tandem, dialling them over -and over until they clog. That's very crude and low-tech, -but it's still a serious business. - -The time had come for action. It was time to take stern measures -with the underground. It was time to start picking up the dropped threads, -the loose edges, the bits of braggadocio here and there; it was time to get -on the stick and start putting serious casework together. Hackers weren't -"invisible." They THOUGHT they were invisible; but the truth was, -they had just been tolerated too long. - -Under sustained police attention in the summer of '89, the digital -underground began to unravel as never before. - -The first big break in the case came very early on: July 1989, -the following month. The perpetrator of the "Tina" switch was caught, -and confessed. His name was "Fry Guy," a 16-year-old in Indiana. -Fry Guy had been a very wicked young man. - -Fry Guy had earned his handle from a stunt involving French fries. -Fry Guy had filched the log-in of a local MacDonald's manager -and had logged-on to the MacDonald's mainframe on the Sprint -Telenet system. Posing as the manager, Fry Guy had altered -MacDonald's records, and given some teenage hamburger-flipping -friends of his, generous raises. He had not been caught. - -Emboldened by success, Fry Guy moved on to credit-card abuse. -Fry Guy was quite an accomplished talker; with a gift for -"social engineering." If you can do "social engineering" ---fast-talk, fake-outs, impersonation, conning, scamming-- -then card abuse comes easy. (Getting away with it in -the long run is another question). - -Fry Guy had run across "Urvile" of the Legion of Doom -on the ALTOS Chat board in Bonn, Germany. ALTOS Chat -was a sophisticated board, accessible through globe-spanning -computer networks like BITnet, Tymnet, and Telenet. -ALTOS was much frequented by members of Germany's -Chaos Computer Club. Two Chaos hackers who hung out on ALTOS, -"Jaeger" and "Pengo," had been the central villains of -Clifford Stoll's Cuckoo's Egg case: consorting in East Berlin -with a spymaster from the KGB, and breaking into American -computers for hire, through the Internet. - -When LoD members learned the story of Jaeger's depredations -from Stoll's book, they were rather less than impressed, -technically speaking. On LoD's own favorite board of the moment, -"Black Ice," LoD members bragged that they themselves could have done -all the Chaos break-ins in a week flat! Nevertheless, LoD were grudgingly -impressed by the Chaos rep, the sheer hairy-eyed daring of hash-smoking -anarchist hackers who had rubbed shoulders with the fearsome big-boys -of international Communist espionage. LoD members sometimes traded -bits of knowledge with friendly German hackers on ALTOS--phone numbers -for vulnerable VAX/VMS computers in Georgia, for instance. -Dutch and British phone phreaks, and the Australian clique of -"Phoenix," "Nom," and "Electron," were ALTOS regulars, too. -In underground circles, to hang out on ALTOS was considered -the sign of an elite dude, a sophisticated hacker of the -international digital jet-set. - -Fry Guy quickly learned how to raid information from credit-card -consumer-reporting agencies. He had over a hundred stolen credit-card -numbers in his notebooks, and upwards of a thousand swiped long-distance -access codes. He knew how to get onto Altos, and how to talk the talk of -the underground convincingly. He now wheedled knowledge of switching-station -tricks from Urvile on the ALTOS system. - -Combining these two forms of knowledge enabled Fry Guy to bootstrap -his way up to a new form of wire-fraud. First, he'd snitched credit card -numbers from credit-company computers. The data he copied included names, -addresses and phone numbers of the random card-holders. - -Then Fry Guy, impersonating a card-holder, called up Western Union -and asked for a cash advance on "his" credit card. Western Union, -as a security guarantee, would call the customer back, at home, -to verify the transaction. - -But, just as he had switched the Florida probation office to "Tina" -in New York, Fry Guy switched the card-holder's number to a local pay-phone. -There he would lurk in wait, muddying his trail by routing and re-routing -the call, through switches as far away as Canada. When the call came through, -he would boldly "social-engineer," or con, the Western Union people, pretending -to be the legitimate card-holder. Since he'd answered the proper phone number, -the deception was not very hard. Western Union's money was then shipped to -a confederate of Fry Guy's in his home town in Indiana. - -Fry Guy and his cohort, using LoD techniques, stole six thousand dollars -from Western Union between December 1988 and July 1989. They also dabbled -in ordering delivery of stolen goods through card-fraud. Fry Guy -was intoxicated with success. The sixteen-year-old fantasized wildly -to hacker rivals, boasting that he'd used rip-off money to hire himself -a big limousine, and had driven out-of-state with a groupie from -his favorite heavy-metal band, Motley Crue. - -Armed with knowledge, power, and a gratifying stream of free money, -Fry Guy now took it upon himself to call local representatives -of Indiana Bell security, to brag, boast, strut, and utter -tormenting warnings that his powerful friends in the notorious -Legion of Doom could crash the national telephone network. -Fry Guy even named a date for the scheme: the Fourth of July, -a national holiday. - -This egregious example of the begging-for-arrest syndrome was shortly -followed by Fry Guy's arrest. After the Indiana telephone company figured -out who he was, the Secret Service had DNRs--Dialed Number Recorders-- -installed on his home phone lines. These devices are not taps, and can't -record the substance of phone calls, but they do record the phone numbers -of all calls going in and out. Tracing these numbers showed Fry Guy's -long-distance code fraud, his extensive ties to pirate bulletin boards, -and numerous personal calls to his LoD friends in Atlanta. By July 11, -1989, Prophet, Urvile and Leftist also had Secret Service DNR -"pen registers" installed on their own lines. - -The Secret Service showed up in force at Fry Guy's house on July 22, 1989, -to the horror of his unsuspecting parents. The raiders were led by -a special agent from the Secret Service's Indianapolis office. -However, the raiders were accompanied and advised by Timothy M. Foley -of the Secret Service's Chicago office (a gentleman about whom -we will soon be hearing a great deal). - -Following federal computer-crime techniques that had been standard -since the early 1980s, the Secret Service searched the house thoroughly, -and seized all of Fry Guy's electronic equipment and notebooks. -All Fry Guy's equipment went out the door in the custody of the -Secret Service, which put a swift end to his depredations. - -The USSS interrogated Fry Guy at length. His case was put in the charge -of Deborah Daniels, the federal US Attorney for the Southern District -of Indiana. Fry Guy was charged with eleven counts of computer fraud, -unauthorized computer access, and wire fraud. The evidence was thorough -and irrefutable. For his part, Fry Guy blamed his corruption on the -Legion of Doom and offered to testify against them. - -Fry Guy insisted that the Legion intended to crash the phone system -on a national holiday. And when AT&T crashed on Martin Luther King Day, -1990, this lent a credence to his claim that genuinely alarmed telco -security and the Secret Service. - -Fry Guy eventually pled guilty on May 31, 1990. On September 14, -he was sentenced to forty-four months' probation and four hundred hours' -community service. He could have had it much worse; but it made sense -to prosecutors to take it easy on this teenage minor, while zeroing -in on the notorious kingpins of the Legion of Doom. - -But the case against LoD had nagging flaws. Despite the best effort -of investigators, it was impossible to prove that the Legion had crashed -the phone system on January 15, because they, in fact, hadn't done so. -The investigations of 1989 did show that certain members of -the Legion of Doom had achieved unprecedented power over the telco -switching stations, and that they were in active conspiracy -to obtain more power yet. Investigators were privately convinced -that the Legion of Doom intended to do awful things with this knowledge, -but mere evil intent was not enough to put them in jail. - -And although the Atlanta Three--Prophet, Leftist, and especially Urvile-- -had taught Fry Guy plenty, they were not themselves credit-card fraudsters. -The only thing they'd "stolen" was long-distance service--and since they'd -done much of that through phone-switch manipulation, there was no easy way -to judge how much they'd "stolen," or whether this practice was even "theft" -of any easily recognizable kind. - -Fry Guy's theft of long-distance codes had cost the phone companies plenty. -The theft of long-distance service may be a fairly theoretical "loss," -but it costs genuine money and genuine time to delete all those stolen codes, -and to re-issue new codes to the innocent owners of those corrupted codes. -The owners of the codes themselves are victimized, and lose time and money -and peace of mind in the hassle. And then there were the credit-card victims -to deal with, too, and Western Union. When it came to rip-off, Fry Guy was -far more of a thief than LoD. It was only when it came to actual computer -expertise that Fry Guy was small potatoes. - -The Atlanta Legion thought most "rules" of cyberspace were for rodents -and losers, but they DID have rules. THEY NEVER CRASHED ANYTHING, -AND THEY NEVER TOOK MONEY. These were rough rules-of-thumb, and -rather dubious principles when it comes to the ethical subtleties -of cyberspace, but they enabled the Atlanta Three to operate with -a relatively clear conscience (though never with peace of mind). - -If you didn't hack for money, if you weren't robbing people of actual funds ---money in the bank, that is-- then nobody REALLY got hurt, in LoD's opinion. -"Theft of service" was a bogus issue, and "intellectual property" was -a bad joke. But LoD had only elitist contempt for rip-off artists, -"leechers," thieves. They considered themselves clean. In their opinion, -if you didn't smash-up or crash any systems --(well, not on purpose, anyhow-- -accidents can happen, just ask Robert Morris) then it was very unfair -to call you a "vandal" or a "cracker." When you were hanging out on-line -with your "pals" in telco security, you could face them down from the higher -plane of hacker morality. And you could mock the police from the supercilious -heights of your hacker's quest for pure knowledge. - -But from the point of view of law enforcement and telco security, however, -Fry Guy was not really dangerous. The Atlanta Three WERE dangerous. -It wasn't the crimes they were committing, but the DANGER, -the potential hazard, the sheer TECHNICAL POWER LoD had accumulated, -that had made the situation untenable. Fry Guy was not LoD. -He'd never laid eyes on anyone in LoD; his only contacts with them -had been electronic. Core members of the Legion of Doom tended to meet -physically for conventions every year or so, to get drunk, give each other -the hacker high-sign, send out for pizza and ravage hotel suites. -Fry Guy had never done any of this. Deborah Daniels assessed Fry Guy -accurately as "an LoD wannabe." - -Nevertheless Fry Guy's crimes would be directly attributed to LoD -in much future police propaganda. LoD would be described as -"a closely knit group" involved in "numerous illegal activities" -including "stealing and modifying individual credit histories," -and "fraudulently obtaining money and property." Fry Guy did this, -but the Atlanta Three didn't; they simply weren't into theft, -but rather intrusion. This caused a strange kink in -the prosecution's strategy. LoD were accused of -"disseminating information about attacking computers -to other computer hackers in an effort to shift the focus -of law enforcement to those other hackers and away from the Legion of Doom." - -This last accusation (taken directly from a press release by the Chicago -Computer Fraud and Abuse Task Force) sounds particularly far-fetched. -One might conclude at this point that investigators would have been -well-advised to go ahead and "shift their focus" from the "Legion of Doom." -Maybe they SHOULD concentrate on "those other hackers"--the ones who were -actually stealing money and physical objects. - -But the Hacker Crackdown of 1990 was not a simple policing action. -It wasn't meant just to walk the beat in cyberspace--it was a CRACKDOWN, -a deliberate attempt to nail the core of the operation, to send a dire -and potent message that would settle the hash of the digital underground -for good. - -By this reasoning, Fry Guy wasn't much more than the electronic equivalent -of a cheap streetcorner dope dealer. As long as the masterminds of LoD were -still flagrantly operating, pushing their mountains of illicit knowledge -right and left, and whipping up enthusiasm for blatant lawbreaking, -then there would be an INFINITE SUPPLY of Fry Guys. - -Because LoD were flagrant, they had left trails everywhere, -to be picked up by law enforcement in New York, Indiana, -Florida, Texas, Arizona, Missouri, even Australia. -But 1990's war on the Legion of Doom was led out of Illinois, -by the Chicago Computer Fraud and Abuse Task Force. - -# - -The Computer Fraud and Abuse Task Force, led by federal prosecutor -William J. Cook, had started in 1987 and had swiftly become one -of the most aggressive local "dedicated computer-crime units." -Chicago was a natural home for such a group. The world's first -computer bulletin-board system had been invented in Illinois. -The state of Illinois had some of the nation's first and sternest -computer crime laws. Illinois State Police were markedly alert -to the possibilities of white-collar crime and electronic fraud. - -And William J. Cook in particular was a rising star in -electronic crime-busting. He and his fellow federal prosecutors -at the U.S. Attorney's office in Chicago had a tight relation -with the Secret Service, especially go-getting Chicago-based agent -Timothy Foley. While Cook and his Department of Justice colleagues -plotted strategy, Foley was their man on the street. - -Throughout the 1980s, the federal government had given prosecutors -an armory of new, untried legal tools against computer crime. -Cook and his colleagues were pioneers in the use of these new statutes -in the real-life cut-and-thrust of the federal courtroom. - -On October 2, 1986, the US Senate had passed the -"Computer Fraud and Abuse Act" unanimously, but there -were pitifully few convictions under this statute. -Cook's group took their name from this statute, -since they were determined to transform this powerful but -rather theoretical Act of Congress into a real-life engine -of legal destruction against computer fraudsters and scofflaws. - -It was not a question of merely discovering crimes, -investigating them, and then trying and punishing their -perpetrators. The Chicago unit, like most everyone else -in the business, already KNEW who the bad guys were: -the Legion of Doom and the writers and editors of Phrack. -The task at hand was to find some legal means of putting -these characters away. - -This approach might seem a bit dubious, to someone not -acquainted with the gritty realities of prosecutorial work. -But prosecutors don't put people in jail for crimes -they have committed; they put people in jail for crimes -they have committed THAT CAN BE PROVED IN COURT. -Chicago federal police put Al Capone in prison -for income-tax fraud. Chicago is a big town, -with a rough-and-ready bare-knuckle tradition -on both sides of the law. - -Fry Guy had broken the case wide open and alerted telco security -to the scope of the problem. But Fry Guy's crimes would not -put the Atlanta Three behind bars--much less the wacko underground -journalists of Phrack. So on July 22, 1989, the same day that -Fry Guy was raided in Indiana, the Secret Service descended upon -the Atlanta Three. - -This was likely inevitable. By the summer of 1989, law enforcement -were closing in on the Atlanta Three from at least six directions at once. -First, there were the leads from Fry Guy, which had led to the DNR registers -being installed on the lines of the Atlanta Three. The DNR evidence alone -would have finished them off, sooner or later. - -But second, the Atlanta lads were already well-known to Control-C -and his telco security sponsors. LoD's contacts with telco security -had made them overconfident and even more boastful than usual; -they felt that they had powerful friends in high places, -and that they were being openly tolerated by telco security. -But BellSouth's Intrusion Task Force were hot on the trail of LoD -and sparing no effort or expense. - -The Atlanta Three had also been identified by name and listed -on the extensive anti-hacker files maintained, and retailed for pay, -by private security operative John Maxfield of Detroit. -Maxfield, who had extensive ties to telco security -and many informants in the underground, was a bete noire -of the Phrack crowd, and the dislike was mutual. - - -The Atlanta Three themselves had written articles for Phrack. -This boastful act could not possibly escape telco and law enforcement -attention. - -"Knightmare," a high-school age hacker from Arizona, -was a close friend and disciple of Atlanta LoD, -but he had been nabbed by the formidable Arizona -Organized Crime and Racketeering Unit. Knightmare was -on some of LoD's favorite boards--"Black Ice" in particular-- -and was privy to their secrets. And to have Gail Thackeray, -the Assistant Attorney General of Arizona, on one's trail -was a dreadful peril for any hacker. - -And perhaps worst of all, Prophet had committed a major blunder -by passing an illicitly copied BellSouth computer-file to Knight Lightning, -who had published it in Phrack. This, as we will see, was an act of dire -consequence for almost everyone concerned. - -On July 22, 1989, the Secret Service showed up at the Leftist's house, -where he lived with his parents. A massive squad of some twenty officers -surrounded the building: Secret Service, federal marshals, local police, -possibly BellSouth telco security; it was hard to tell in the crush. -Leftist's dad, at work in his basement office, first noticed -a muscular stranger in plain clothes crashing through the -back yard with a drawn pistol. As more strangers poured -into the house, Leftist's dad naturally assumed there was -an armed robbery in progress. - -Like most hacker parents, Leftist's mom and dad had only the vaguest -notions of what their son had been up to all this time. Leftist had -a day-job repairing computer hardware. His obsession with computers -seemed a bit odd, but harmless enough, and likely to produce a well- -paying career. The sudden, overwhelming raid left Leftist's -parents traumatized. - -The Leftist himself had been out after work with his co-workers, -surrounding a couple of pitchers of margaritas. As he came trucking -on tequila-numbed feet up the pavement, toting a bag full of floppy-disks, -he noticed a large number of unmarked cars parked in his driveway. -All the cars sported tiny microwave antennas. - -The Secret Service had knocked the front door off its hinges, -almost flattening his mom. - -Inside, Leftist was greeted by Special Agent James Cool -of the US Secret Service, Atlanta office. Leftist was flabbergasted. -He'd never met a Secret Service agent before. He could not imagine -that he'd ever done anything worthy of federal attention. -He'd always figured that if his activities became intolerable, -one of his contacts in telco security would give him a private -phone-call and tell him to knock it off. - -But now Leftist was pat-searched for weapons by grim professionals, -and his bag of floppies was quickly seized. He and his parents were -all shepherded into separate rooms and grilled at length as a score -of officers scoured their home for anything electronic. - -Leftist was horrified as his treasured IBM AT personal computer -with its forty-meg hard disk, and his recently purchased 80386 IBM-clone -with a whopping hundred-meg hard disk, both went swiftly out the door -in Secret Service custody. They also seized all his disks, all his notebooks, -and a tremendous booty in dogeared telco documents that Leftist had snitched -out of trash dumpsters. - -Leftist figured the whole thing for a big misunderstanding. -He'd never been into MILITARY computers. He wasn't a SPY or a COMMUNIST. -He was just a good ol' Georgia hacker, and now he just wanted all these -people out of the house. But it seemed they wouldn't go until he made -some kind of statement. - -And so, he levelled with them. - -And that, Leftist said later from his federal prison camp in Talladega, -Alabama, was a big mistake. The Atlanta area was unique, -in that it had three members of the Legion of Doom who actually -occupied more or less the same physical locality. Unlike the rest -of LoD, who tended to associate by phone and computer, -Atlanta LoD actually WERE "tightly knit." It was no real -surprise that the Secret Service agents apprehending Urvile -at the computer-labs at Georgia Tech, would discover Prophet -with him as well. - -Urvile, a 21-year-old Georgia Tech student in polymer chemistry, -posed quite a puzzling case for law enforcement. Urvile--also known -as "Necron 99," as well as other handles, for he tended to change his -cover-alias about once a month--was both an accomplished hacker -and a fanatic simulation-gamer. - -Simulation games are an unusual hobby; but then hackers are unusual people, -and their favorite pastimes tend to be somewhat out of the ordinary. -The best-known American simulation game is probably "Dungeons & Dragons," -a multi-player parlor entertainment played with paper, maps, pencils, -statistical tables and a variety of oddly-shaped dice. Players pretend -to be heroic characters exploring a wholly-invented fantasy world. -The fantasy worlds of simulation gaming are commonly pseudo-medieval, -involving swords and sorcery--spell-casting wizards, knights in armor, -unicorns and dragons, demons and goblins. - -Urvile and his fellow gamers preferred their fantasies highly technological. -They made use of a game known as "G.U.R.P.S.," the "Generic Universal Role -Playing System," published by a company called Steve Jackson Games (SJG). - -"G.U.R.P.S." served as a framework for creating a wide variety of artificial -fantasy worlds. Steve Jackson Games published a smorgasboard of books, -full of detailed information and gaming hints, which were used to flesh-out -many different fantastic backgrounds for the basic GURPS framework. -Urvile made extensive use of two SJG books called GURPS High-Tech -and GURPS Special Ops. - -In the artificial fantasy-world of GURPS Special Ops, -players entered a modern fantasy of intrigue and international espionage. -On beginning the game, players started small and powerless, -perhaps as minor-league CIA agents or penny-ante arms dealers. -But as players persisted through a series of game sessions -(game sessions generally lasted for hours, over long, -elaborate campaigns that might be pursued for months on end) -then they would achieve new skills, new knowledge, new power. -They would acquire and hone new abilities, such as marksmanship, -karate, wiretapping, or Watergate burglary. They could also win -various kinds of imaginary booty, like Berettas, or martini shakers, -or fast cars with ejection seats and machine-guns under the headlights. - -As might be imagined from the complexity of these games, -Urvile's gaming notes were very detailed and extensive. -Urvile was a "dungeon-master," inventing scenarios -for his fellow gamers, giant simulated adventure-puzzles -for his friends to unravel. Urvile's game notes covered -dozens of pages with all sorts of exotic lunacy, all about -ninja raids on Libya and break-ins on encrypted Red Chinese supercomputers. -His notes were written on scrap-paper and kept in loose-leaf binders. - -The handiest scrap paper around Urvile's college digs were the many pounds of -BellSouth printouts and documents that he had snitched out of telco dumpsters. -His notes were written on the back of misappropriated telco property. -Worse yet, the gaming notes were chaotically interspersed with Urvile's -hand-scrawled records involving ACTUAL COMPUTER INTRUSIONS that he -had committed. - -Not only was it next to impossible to tell Urvile's fantasy game-notes -from cyberspace "reality," but Urvile himself barely made this distinction. -It's no exaggeration to say that to Urvile it was ALL a game. Urvile was -very bright, highly imaginative, and quite careless of other people's notions -of propriety. His connection to "reality" was not something to which he paid -a great deal of attention. - -Hacking was a game for Urvile. It was an amusement he was carrying out, -it was something he was doing for fun. And Urvile was an obsessive young man. -He could no more stop hacking than he could stop in the middle of -a jigsaw puzzle, or stop in the middle of reading a Stephen Donaldson -fantasy trilogy. (The name "Urvile" came from a best-selling Donaldson novel.) - -Urvile's airy, bulletproof attitude seriously annoyed his interrogators. -First of all, he didn't consider that he'd done anything wrong. -There was scarcely a shred of honest remorse in him. On the contrary, -he seemed privately convinced that his police interrogators were operating -in a demented fantasy-world all their own. Urvile was too polite -and well-behaved to say this straight-out, but his reactions were askew -and disquieting. - -For instance, there was the business about LoD's ability -to monitor phone-calls to the police and Secret Service. -Urvile agreed that this was quite possible, and posed -no big problem for LoD. In fact, he and his friends -had kicked the idea around on the "Black Ice" board, -much as they had discussed many other nifty notions, -such as building personal flame-throwers and jury-rigging -fistfulls of blasting-caps. They had hundreds of dial-up numbers -for government agencies that they'd gotten through scanning Atlanta phones, -or had pulled from raided VAX/VMS mainframe computers. - -Basically, they'd never gotten around to listening in on the cops -because the idea wasn't interesting enough to bother with. -Besides, if they'd been monitoring Secret Service phone calls, -obviously they'd never have been caught in the first place. Right? - -The Secret Service was less than satisfied with this rapier-like hacker logic. - -Then there was the issue of crashing the phone system. No problem, -Urvile admitted sunnily. Atlanta LoD could have shut down phone service -all over Atlanta any time they liked. EVEN THE 911 SERVICE? -Nothing special about that, Urvile explained patiently. -Bring the switch to its knees, with say the UNIX "makedir" bug, -and 911 goes down too as a matter of course. The 911 system -wasn't very interesting, frankly. It might be tremendously -interesting to cops (for odd reasons of their own), but as -technical challenges went, the 911 service was yawnsville. - -So of course the Atlanta Three could crash service. -They probably could have crashed service all over -BellSouth territory, if they'd worked at it for a while. -But Atlanta LoD weren't crashers. Only losers and rodents -were crashers. LoD were ELITE. - -Urvile was privately convinced that sheer technical -expertise could win him free of any kind of problem. -As far as he was concerned, elite status in the digital -underground had placed him permanently beyond the intellectual -grasp of cops and straights. Urvile had a lot to learn. - -Of the three LoD stalwarts, Prophet was in the most direct trouble. -Prophet was a UNIX programming expert who burrowed in and out -of the Internet as a matter of course. He'd started his hacking -career at around age 14, meddling with a UNIX mainframe system -at the University of North Carolina. - -Prophet himself had written the handy Legion of Doom -file "UNIX Use and Security From the Ground Up." -UNIX (pronounced "you-nicks") is a powerful, -flexible computer operating-system, for multi-user, -multi-tasking computers. In 1969, when UNIX was created -in Bell Labs, such computers were exclusive to large -corporations and universities, but today UNIX is run -on thousands of powerful home machines. UNIX was -particularly well-suited to telecommunications programming, -and had become a standard in the field. Naturally, UNIX -also became a standard for the elite hacker and phone phreak. -Lately, Prophet had not been so active as Leftist and Urvile, -but Prophet was a recidivist. In 1986, when he was eighteen, -Prophet had been convicted of "unauthorized access -to a computer network" in North Carolina. He'd been -discovered breaking into the Southern Bell Data Network, -a UNIX-based internal telco network supposedly closed to the public. -He'd gotten a typical hacker sentence: six months suspended, -120 hours community service, and three years' probation. - -After that humiliating bust, Prophet had gotten rid of most of his -tonnage of illicit phreak and hacker data, and had tried to go straight. -He was, after all, still on probation. But by the autumn of 1988, -the temptations of cyberspace had proved too much for young Prophet, -and he was shoulder-to-shoulder with Urvile and Leftist into some -of the hairiest systems around. - -In early September 1988, he'd broken into BellSouth's centralized -automation system, AIMSX or "Advanced Information Management System." -AIMSX was an internal business network for BellSouth, where telco -employees stored electronic mail, databases, memos, and calendars, -and did text processing. Since AIMSX did not have public dial-ups, -it was considered utterly invisible to the public, and was not well-secured ---it didn't even require passwords. Prophet abused an account known -as "waa1," the personal account of an unsuspecting telco employee. -Disguised as the owner of waa1, Prophet made about ten visits to AIMSX. - -Prophet did not damage or delete anything in the system. -His presence in AIMSX was harmless and almost invisible. -But he could not rest content with that. - -One particular piece of processed text on AIMSX was a telco document -known as "Bell South Standard Practice 660-225-104SV Control Office -Administration of Enhanced 911 Services for Special Services -and Major Account Centers dated March 1988." - -Prophet had not been looking for this document. It was merely one -among hundreds of similar documents with impenetrable titles. -However, having blundered over it in the course of his illicit -wanderings through AIMSX, he decided to take it with him as a trophy. -It might prove very useful in some future boasting, bragging, -and strutting session. So, some time in September 1988, -Prophet ordered the AIMSX mainframe computer to copy this document -(henceforth called simply called "the E911 Document") and to transfer -this copy to his home computer. - -No one noticed that Prophet had done this. He had "stolen" -the E911 Document in some sense, but notions of property -in cyberspace can be tricky. BellSouth noticed nothing wrong, -because BellSouth still had their original copy. They had not -been "robbed" of the document itself. Many people were supposed -to copy this document--specifically, people who worked for the -nineteen BellSouth "special services and major account centers," -scattered throughout the Southeastern United States. That was -what it was for, why it was present on a computer network -in the first place: so that it could be copied and read-- -by telco employees. But now the data had been copied -by someone who wasn't supposed to look at it. - -Prophet now had his trophy. But he further decided to store -yet another copy of the E911 Document on another person's computer. -This unwitting person was a computer enthusiast named Richard Andrews -who lived near Joliet, Illinois. Richard Andrews was a UNIX programmer -by trade, and ran a powerful UNIX board called "Jolnet," in the basement -of his house. - -Prophet, using the handle "Robert Johnson," had obtained an account -on Richard Andrews' computer. And there he stashed the E911 Document, -by storing it in his own private section of Andrews' computer. - -Why did Prophet do this? If Prophet had eliminated the E911 Document -from his own computer, and kept it hundreds of miles away, on another machine, under an -alias, then he might have been fairly safe from discovery and prosecution-- -although his sneaky action had certainly put the unsuspecting Richard Andrews -at risk. - -But, like most hackers, Prophet was a pack-rat for illicit data. -When it came to the crunch, he could not bear to part from his trophy. -When Prophet's place in Decatur, Georgia was raided in July 1989, -there was the E911 Document, a smoking gun. And there was Prophet -in the hands of the Secret Service, doing his best to "explain." - -Our story now takes us away from the Atlanta Three and their raids -of the Summer of 1989. We must leave Atlanta Three "cooperating fully" -with their numerous investigators. And all three of them did cooperate, -as their Sentencing Memorandum from the US District Court of the -Northern Division of Georgia explained--just before all three of them -were sentenced to various federal prisons in November 1990. - -We must now catch up on the other aspects of the war on the Legion of Doom. -The war on the Legion was a war on a network--in fact, a network of three -networks, which intertwined and interrelated in a complex fashion. -The Legion itself, with Atlanta LoD, and their hanger-on Fry Guy, -were the first network. The second network was Phrack magazine, -with its editors and contributors. - -The third network involved the electronic circle around a hacker -known as "Terminus." - -The war against these hacker networks was carried out by -a law enforcement network. Atlanta LoD and Fry Guy -were pursued by USSS agents and federal prosecutors in Atlanta, -Indiana, and Chicago. "Terminus" found himself pursued by USSS -and federal prosecutors from Baltimore and Chicago. And the war -against Phrack was almost entirely a Chicago operation. - -The investigation of Terminus involved a great deal of energy, -mostly from the Chicago Task Force, but it was to be the least-known -and least-publicized of the Crackdown operations. Terminus, who lived -in Maryland, was a UNIX programmer and consultant, fairly well-known -(under his given name) in the UNIX community, as an acknowledged expert -on AT&T minicomputers. Terminus idolized AT&T, especially Bellcore, -and longed for public recognition as a UNIX expert; his highest ambition -was to work for Bell Labs. - -But Terminus had odd friends and a spotted history. -Terminus had once been the subject of an admiring interview -in Phrack (Volume II, Issue 14, Phile 2--dated May 1987). -In this article, Phrack co-editor Taran King described -"Terminus" as an electronics engineer, 5'9", brown-haired, -born in 1959--at 28 years old, quite mature for a hacker. - -Terminus had once been sysop of a phreak/hack underground board -called "MetroNet," which ran on an Apple II. Later he'd replaced -"MetroNet" with an underground board called "MegaNet," -specializing in IBMs. In his younger days, Terminus had written -one of the very first and most elegant code-scanning programs -for the IBM-PC. This program had been widely distributed -in the underground. Uncounted legions of PC-owning phreaks and -hackers had used Terminus's scanner program to rip-off telco codes. -This feat had not escaped the attention of telco security; -it hardly could, since Terminus's earlier handle, "Terminal Technician," -was proudly written right on the program. - -When he became a full-time computer professional -(specializing in telecommunications programming), -he adopted the handle Terminus, meant to indicate that he -had "reached the final point of being a proficient hacker." -He'd moved up to the UNIX-based "Netsys" board on an AT&T computer, -with four phone lines and an impressive 240 megs of storage. -"Netsys" carried complete issues of Phrack, and Terminus was -quite friendly with its publishers, Taran King and Knight Lightning. - -In the early 1980s, Terminus had been a regular on Plovernet, -Pirate-80, Sherwood Forest and Shadowland, all well-known pirate boards, -all heavily frequented by the Legion of Doom. As it happened, Terminus -was never officially "in LoD," because he'd never been given the official -LoD high-sign and back-slap by Legion maven Lex Luthor. Terminus had -never physically met anyone from LoD. But that scarcely mattered much-- -the Atlanta Three themselves had never been officially vetted by Lex, either. - -As far as law enforcement was concerned, the issues were clear. -Terminus was a full-time, adult computer professional -with particular skills at AT&T software and hardware-- -but Terminus reeked of the Legion of Doom and the underground. - -On February 1, 1990--half a month after the Martin Luther King Day Crash-- -USSS agents Tim Foley from Chicago, and Jack Lewis from the Baltimore office, -accompanied by AT&T security officer Jerry Dalton, travelled to Middle Town, -Maryland. There they grilled Terminus in his home (to the stark terror of -his wife and small children), and, in their customary fashion, hauled his -computers out the door. - -The Netsys machine proved to contain a plethora of arcane UNIX software-- -proprietary source code formally owned by AT&T. Software such as: -UNIX System Five Release 3.2; UNIX SV Release 3.1; UUCP communications -software; KORN SHELL; RFS; IWB; WWB; DWB; the C++ programming language; -PMON; TOOL CHEST; QUEST; DACT, and S FIND. - -In the long-established piratical tradition of the underground, -Terminus had been trading this illicitly-copied software with -a small circle of fellow UNIX programmers. Very unwisely, -he had stored seven years of his electronic mail on his Netsys machine, -which documented all the friendly arrangements he had made with -his various colleagues. - -Terminus had not crashed the AT&T phone system on January 15. -He was, however, blithely running a not-for-profit AT&T -software-piracy ring. This was not an activity AT&T found amusing. -AT&T security officer Jerry Dalton valued this "stolen" property -at over three hundred thousand dollars. - -AT&T's entry into the tussle of free enterprise had been complicated -by the new, vague groundrules of the information economy. -Until the break-up of Ma Bell, AT&T was forbidden to sell -computer hardware or software. Ma Bell was the phone company; -Ma Bell was not allowed to use the enormous revenue from -telephone utilities, in order to finance any entry into -the computer market. - -AT&T nevertheless invented the UNIX operating system. -And somehow AT&T managed to make UNIX a minor source of income. -Weirdly, UNIX was not sold as computer software, -but actually retailed under an obscure regulatory -exemption allowing sales of surplus equipment and scrap. -Any bolder attempt to promote or retail UNIX would have -aroused angry legal opposition from computer companies. -Instead, UNIX was licensed to universities, at modest rates, -where the acids of academic freedom ate away steadily at AT&T's -proprietary rights. - -Come the breakup, AT&T recognized that UNIX was a potential gold-mine. -By now, large chunks of UNIX code had been created that were not AT&T's, -and were being sold by others. An entire rival UNIX-based operating system -had arisen in Berkeley, California (one of the world's great founts of -ideological hackerdom). Today, "hackers" commonly consider "Berkeley UNIX" -to be technically superior to AT&T's "System V UNIX," but AT&T has not -allowed mere technical elegance to intrude on the real-world business -of marketing proprietary software. AT&T has made its own code deliberately -incompatible with other folks' UNIX, and has written code that it can prove -is copyrightable, even if that code happens to be somewhat awkward--"kludgey." -AT&T UNIX user licenses are serious business agreements, replete with very -clear copyright statements and non-disclosure clauses. - -AT&T has not exactly kept the UNIX cat in the bag, -but it kept a grip on its scruff with some success. -By the rampant, explosive standards of software piracy, -AT&T UNIX source code is heavily copyrighted, well-guarded, -well-licensed. UNIX was traditionally run only on -mainframe machines, owned by large groups of suit-and-tie -professionals, rather than on bedroom machines where -people can get up to easy mischief. - -And AT&T UNIX source code is serious high-level programming. -The number of skilled UNIX programmers with any actual motive -to swipe UNIX source code is small. It's tiny, compared to -the tens of thousands prepared to rip-off, say, entertaining -PC games like "Leisure Suit Larry." - -But by 1989, the warez-d00d underground, in the persons of Terminus -and his friends, was gnawing at AT&T UNIX. And the property in question -was not sold for twenty bucks over the counter at the local branch of -Babbage's or Egghead's; this was massive, sophisticated, multi-line, -multi-author corporate code worth tens of thousands of dollars. - -It must be recognized at this point that Terminus's purported ring of UNIX -software pirates had not actually made any money from their suspected crimes. -The $300,000 dollar figure bandied about for the contents of Terminus's -computer did not mean that Terminus was in actual illicit possession -of three hundred thousand of AT&T's dollars. Terminus was shipping -software back and forth, privately, person to person, for free. -He was not making a commercial business of piracy. He hadn't -asked for money; he didn't take money. He lived quite modestly. - -AT&T employees--as well as freelance UNIX consultants, like Terminus-- -commonly worked with "proprietary" AT&T software, both in the office -and at home on their private machines. AT&T rarely sent security officers -out to comb the hard disks of its consultants. Cheap freelance UNIX -contractors were quite useful to AT&T; they didn't have health insurance -or retirement programs, much less union membership in the Communication -Workers of America. They were humble digital drudges, wandering with mop -and bucket through the Great Technological Temple of AT&T; but when the -Secret Service arrived at their homes, it seemed they were eating with -company silverware and sleeping on company sheets! Outrageously, they -behaved as if the things they worked with every day belonged to them! - -And these were no mere hacker teenagers with their hands full -of trash-paper and their noses pressed to the corporate windowpane. -These guys were UNIX wizards, not only carrying AT&T data in their -machines and their heads, but eagerly networking about it, -over machines that were far more powerful than anything previously -imagined in private hands. How do you keep people disposable, -yet assure their awestruck respect for your property? It was a dilemma. - -Much UNIX code was public-domain, available for free. Much "proprietary" -UNIX code had been extensively re-written, perhaps altered so much that it -became an entirely new product--or perhaps not. Intellectual property rights -for software developers were, and are, extraordinarily complex and confused. -And software "piracy," like the private copying of videos, is one of the most -widely practiced "crimes" in the world today. - -The USSS were not experts in UNIX or familiar with the customs of its use. -The United States Secret Service, considered as a body, did not have one single -person in it who could program in a UNIX environment--no, not even one. -The Secret Service WERE making extensive use of expert help, but the "experts" -they had chosen were AT&T and Bellcore security officials, the very victims of -the purported crimes under investigation, the very people whose interest in -AT&T's "proprietary" software was most pronounced. - -On February 6, 1990, Terminus was arrested by Agent Lewis. -Eventually, Terminus would be sent to prison for his illicit -use of a piece of AT&T software. - -The issue of pirated AT&T software would bubble along in the background -during the war on the Legion of Doom. Some half-dozen of Terminus's on-line -acquaintances, including people in Illinois, Texas and California, -were grilled by the Secret Service in connection with the illicit -copying of software. Except for Terminus, however, none were charged -with a crime. None of them shared his peculiar prominence in the -hacker underground. - -But that did not mean that these people would, or could, -stay out of trouble. The transferral of illicit data in -cyberspace is hazy and ill-defined business, with paradoxical -dangers for everyone concerned: hackers, signal carriers, -board owners, cops, prosecutors, even random passers-by. -Sometimes, well-meant attempts to avert trouble -or punish wrongdoing bring more trouble than -would simple ignorance, indifference or impropriety. - -Terminus's "Netsys" board was not a common-or-garden -bulletin board system, though it had most of the usual -functions of a board. Netsys was not a stand-alone machine, -but part of the globe-spanning "UUCP" cooperative network. -The UUCP network uses a set of Unix software programs called -"Unix-to-Unix Copy," which allows Unix systems to throw data to -one another at high speed through the public telephone network. -UUCP is a radically decentralized, not-for-profit network of UNIX computers. -There are tens of thousands of these UNIX machines. Some are small, -but many are powerful and also link to other networks. UUCP has -certain arcane links to major networks such as JANET, EasyNet, BITNET, -JUNET, VNET, DASnet, PeaceNet and FidoNet, as well as the gigantic Internet. -(The so-called "Internet" is not actually a network itself, but rather an -"internetwork" connections standard that allows several globe-spanning -computer networks to communicate with one another. Readers fascinated -by the weird and intricate tangles of modern computer networks may enjoy -John S. Quarterman's authoritative 719-page explication, The Matrix, -Digital Press, 1990.) - -A skilled user of Terminus' UNIX machine could send and receive -electronic mail from almost any major computer network in the world. -Netsys was not called a "board" per se, but rather a "node." -"Nodes" were larger, faster, and more sophisticated than mere "boards," -and for hackers, to hang out on internationally-connected "nodes" -was quite the step up from merely hanging out on local "boards." - -Terminus's Netsys node in Maryland had a number of direct -links to other, similar UUCP nodes, run by people who shared his -interests and at least something of his free-wheeling attitude. -One of these nodes was Jolnet, owned by Richard Andrews, who, -like Terminus, was an independent UNIX consultant. -Jolnet also ran UNIX, and could be contacted at high speed -by mainframe machines from all over the world. Jolnet was -quite a sophisticated piece of work, technically speaking, -but it was still run by an individual, as a private, -not-for-profit hobby. Jolnet was mostly used by other -UNIX programmers--for mail, storage, and access to networks. -Jolnet supplied access network access to about two hundred people, -as well as a local junior college. - -Among its various features and services, Jolnet also carried -Phrack magazine. - -For reasons of his own, Richard Andrews had become suspicious -of a new user called "Robert Johnson." Richard Andrews -took it upon himself to have a look at what "Robert Johnson" -was storing in Jolnet. And Andrews found the E911 Document. - -"Robert Johnson" was the Prophet from the Legion of Doom, -and the E911 Document was illicitly copied data from Prophet's -raid on the BellSouth computers. - -The E911 Document, a particularly illicit piece of digital property, -was about to resume its long, complex, and disastrous career. - -It struck Andrews as fishy that someone not a telephone employee -should have a document referring to the "Enhanced 911 System." -Besides, the document itself bore an obvious warning. - -"WARNING: NOT FOR USE OR DISCLOSURE OUTSIDE BELLSOUTH -OR ANY OF ITS SUBSIDIARIES EXCEPT UNDER WRITTEN AGREEMENT." - -These standard nondisclosure tags are often appended to all sorts -of corporate material. Telcos as a species are particularly notorious -for stamping most everything in sight as "not for use or disclosure." -Still, this particular piece of data was about the 911 System. -That sounded bad to Rich Andrews. - -Andrews was not prepared to ignore this sort of trouble. -He thought it would be wise to pass the document along -to a friend and acquaintance on the UNIX network, for consultation. -So, around September 1988, Andrews sent yet another copy of the -E911 Document electronically to an AT&T employee, one Charles Boykin, -who ran a UNIX-based node called "attctc" in Dallas, Texas. - -"Attctc" was the property of AT&T, and was run from AT&T's -Customer Technology Center in Dallas, hence the name "attctc." -"Attctc" was better-known as "Killer," the name of the machine -that the system was running on. "Killer" was a hefty, powerful, -AT&T 3B2 500 model, a multi-user, multi-tasking UNIX platform -with 32 meg of memory and a mind-boggling 3.2 Gigabytes of storage. -When Killer had first arrived in Texas, in 1985, the 3B2 had been -one of AT&T's great white hopes for going head-to-head with IBM -for the corporate computer-hardware market. "Killer" had been shipped -to the Customer Technology Center in the Dallas Infomart, essentially -a high-technology mall, and there it sat, a demonstration model. - -Charles Boykin, a veteran AT&T hardware and digital communications expert, -was a local technical backup man for the AT&T 3B2 system. As a display model -in the Infomart mall, "Killer" had little to do, and it seemed a shame -to waste the system's capacity. So Boykin ingeniously wrote some UNIX -bulletin-board software for "Killer," and plugged the machine in to the -local phone network. "Killer's" debut in late 1985 made it the first -publicly available UNIX site in the state of Texas. Anyone who wanted to -play was welcome. - -The machine immediately attracted an electronic community. -It joined the UUCP network, and offered network links -to over eighty other computer sites, all of which became dependent -on Killer for their links to the greater world of cyberspace. -And it wasn't just for the big guys; personal computer users -also stored freeware programs for the Amiga, the Apple, -the IBM and the Macintosh on Killer's vast 3,200 meg archives. -At one time, Killer had the largest library of public-domain -Macintosh software in Texas. - -Eventually, Killer attracted about 1,500 users, -all busily communicating, uploading and downloading, -getting mail, gossipping, and linking to arcane -and distant networks. - -Boykin received no pay for running Killer. He considered -it good publicity for the AT&T 3B2 system (whose sales were -somewhat less than stellar), but he also simply enjoyed -the vibrant community his skill had created. He gave away -the bulletin-board UNIX software he had written, free of charge. - -In the UNIX programming community, Charlie Boykin had the -reputation of a warm, open-hearted, level-headed kind of guy. -In 1989, a group of Texan UNIX professionals voted Boykin -"System Administrator of the Year." He was considered -a fellow you could trust for good advice. - -In September 1988, without warning, the E911 Document -came plunging into Boykin's life, forwarded by Richard Andrews. -Boykin immediately recognized that the Document was hot property. -He was not a voice-communications man, and knew little about -the ins and outs of the Baby Bells, but he certainly knew what -the 911 System was, and he was angry to see confidential data -about it in the hands of a nogoodnik. This was clearly a -matter for telco security. So, on September 21, 1988, Boykin -made yet ANOTHER copy of the E911 Document and passed this -one along to a professional acquaintance of his, one Jerome Dalton, -from AT&T Corporate Information Security. Jerry Dalton was the -very fellow who would later raid Terminus's house. - -From AT&T's security division, the E911 Document went to Bellcore. - -Bellcore (or BELL COmmunications REsearch) had once been the central -laboratory of the Bell System. Bell Labs employees had invented -the UNIX operating system. Now Bellcore was a quasi-independent, -jointly owned company that acted as the research arm for all seven -of the Baby Bell RBOCs. Bellcore was in a good position to co-ordinate -security technology and consultation for the RBOCs, and the gentleman in -charge of this effort was Henry M. Kluepfel, a veteran of the Bell System -who had worked there for twenty-four years. - -On October 13, 1988, Dalton passed the E911 Document to Henry Kluepfel. -Kluepfel, a veteran expert witness in telecommunications fraud and -computer-fraud cases, had certainly seen worse trouble than this. -He recognized the document for what it was: a trophy from a hacker break-in. - -However, whatever harm had been done in the intrusion was presumably old news. -At this point there seemed little to be done. Kluepfel made a careful note -of the circumstances and shelved the problem for the time being. - -Whole months passed. - -February 1989 arrived. The Atlanta Three were living it up -in Bell South's switches, and had not yet met their comeuppance. -The Legion was thriving. So was Phrack magazine. -A good six months had passed since Prophet's AIMSX break-in. -Prophet, as hackers will, grew weary of sitting on his laurels. -"Knight Lightning" and "Taran King," the editors of Phrack, -were always begging Prophet for material they could publish. -Prophet decided that the heat must be off by this time, -and that he could safely brag, boast, and strut. - -So he sent a copy of the E911 Document--yet another one-- -from Rich Andrews' Jolnet machine to Knight Lightning's -BITnet account at the University of Missouri. -Let's review the fate of the document so far. - -0. The original E911 Document. This in the AIMSX system -on a mainframe computer in Atlanta, available to hundreds of people, -but all of them, presumably, BellSouth employees. An unknown number -of them may have their own copies of this document, but they are all -professionals and all trusted by the phone company. - -1. Prophet's illicit copy, at home on his own computer in Decatur, Georgia. - -2. Prophet's back-up copy, stored on Rich Andrew's Jolnet machine - in the basement of Rich Andrews' house near Joliet Illinois. - -3. Charles Boykin's copy on "Killer" in Dallas, Texas, - sent by Rich Andrews from Joliet. - -4. Jerry Dalton's copy at AT&T Corporate Information Security in New Jersey, - sent from Charles Boykin in Dallas. - -5. Henry Kluepfel's copy at Bellcore security headquarters in New Jersey, - sent by Dalton. -6. Knight Lightning's copy, sent by Prophet from Rich Andrews' machine, - and now in Columbia, Missouri. - -We can see that the "security" situation of this proprietary document, -once dug out of AIMSX, swiftly became bizarre. Without any money -changing hands, without any particular special effort, this data -had been reproduced at least six times and had spread itself all over -the continent. By far the worst, however, was yet to come. - -In February 1989, Prophet and Knight Lightning bargained electronically -over the fate of this trophy. Prophet wanted to boast, but, at the same time, -scarcely wanted to be caught. - -For his part, Knight Lightning was eager to publish as much of the document -as he could manage. Knight Lightning was a fledgling political-science major -with a particular interest in freedom-of-information issues. He would gladly -publish most anything that would reflect glory on the prowess of the -underground and embarrass the telcos. However, Knight Lightning himself -had contacts in telco security, and sometimes consulted them on material -he'd received that might be too dicey for publication. - -Prophet and Knight Lightning decided to edit the E911 Document -so as to delete most of its identifying traits. First of all, -its large "NOT FOR USE OR DISCLOSURE" warning had to go. -Then there were other matters. For instance, it listed -the office telephone numbers of several BellSouth 911 -specialists in Florida. If these phone numbers were -published in Phrack, the BellSouth employees involved -would very likely be hassled by phone phreaks, -which would anger BellSouth no end, and pose a -definite operational hazard for both Prophet and Phrack. - -So Knight Lightning cut the Document almost in half, -removing the phone numbers and some of the touchier -and more specific information. He passed it back -electronically to Prophet; Prophet was still nervous, -so Knight Lightning cut a bit more. They finally agreed -that it was ready to go, and that it would be published -in Phrack under the pseudonym, "The Eavesdropper." - -And this was done on February 25, 1989. - -The twenty-fourth issue of Phrack featured a chatty interview -with co-ed phone-phreak "Chanda Leir," three articles on BITNET -and its links to other computer networks, an article on 800 and 900 -numbers by "Unknown User," "VaxCat's" article on telco basics -(slyly entitled "Lifting Ma Bell's Veil of Secrecy,)" and -the usual "Phrack World News." - -The News section, with painful irony, featured an extended account -of the sentencing of "Shadowhawk," an eighteen-year-old Chicago hacker -who had just been put in federal prison by William J. Cook himself. - -And then there were the two articles by "The Eavesdropper." -The first was the edited E911 Document, now titled -"Control Office Administration Of Enhanced 911 Services -for Special Services and Major Account Centers." -Eavesdropper's second article was a glossary of terms -explaining the blizzard of telco acronyms and buzzwords -in the E911 Document. - -The hapless document was now distributed, in the usual Phrack routine, -to a good one hundred and fifty sites. Not a hundred and fifty PEOPLE, -mind you--a hundred and fifty SITES, some of these sites linked to UNIX -nodes or bulletin board systems, which themselves had readerships of tens, -dozens, even hundreds of people. - -This was February 1989. Nothing happened immediately. -Summer came, and the Atlanta crew were raided by the Secret Service. -Fry Guy was apprehended. Still nothing whatever happened to Phrack. -Six more issues of Phrack came out, 30 in all, more or less on -a monthly schedule. Knight Lightning and co-editor Taran King -went untouched. - -Phrack tended to duck and cover whenever the heat came down. -During the summer busts of 1987--(hacker busts tended to cluster in summer, -perhaps because hackers were easier to find at home than in college)-- -Phrack had ceased publication for several months, and laid low. -Several LoD hangers-on had been arrested, but nothing had happened -to the Phrack crew, the premiere gossips of the underground. -In 1988, Phrack had been taken over by a new editor, -"Crimson Death," a raucous youngster with a taste for anarchy files. -1989, however, looked like a bounty year for the underground. -Knight Lightning and his co-editor Taran King took up the reins again, -and Phrack flourished throughout 1989. Atlanta LoD went down hard in -the summer of 1989, but Phrack rolled merrily on. Prophet's E911 Document -seemed unlikely to cause Phrack any trouble. By January 1990, -it had been available in Phrack for almost a year. Kluepfel and Dalton, -officers of Bellcore and AT&T security, had possessed the document -for sixteen months--in fact, they'd had it even before Knight Lightning -himself, and had done nothing in particular to stop its distribution. -They hadn't even told Rich Andrews or Charles Boykin to erase the copies -from their UNIX nodes, Jolnet and Killer. - -But then came the monster Martin Luther King Day Crash of January 15, 1990. - -A flat three days later, on January 18, four agents showed up -at Knight Lightning's fraternity house. One was Timothy Foley, -the second Barbara Golden, both of them Secret Service agents -from the Chicago office. Also along was a University of Missouri -security officer, and Reed Newlin, a security man from Southwestern Bell, -the RBOC having jurisdiction over Missouri. - -Foley accused Knight Lightning of causing the nationwide crash -of the phone system. - -Knight Lightning was aghast at this allegation. On the face of it, -the suspicion was not entirely implausible--though Knight Lightning -knew that he himself hadn't done it. Plenty of hot-dog hackers -had bragged that they could crash the phone system, however. -"Shadowhawk," for instance, the Chicago hacker whom William Cook -had recently put in jail, had several times boasted on boards -that he could "shut down AT&T's public switched network." - -And now this event, or something that looked just like it, -had actually taken place. The Crash had lit a fire under -the Chicago Task Force. And the former fence-sitters at -Bellcore and AT&T were now ready to roll. The consensus -among telco security--already horrified by the skill of -the BellSouth intruders --was that the digital underground -was out of hand. LoD and Phrack must go. And in publishing -Prophet's E911 Document, Phrack had provided law enforcement -with what appeared to be a powerful legal weapon. - -Foley confronted Knight Lightning about the E911 Document. - -Knight Lightning was cowed. He immediately began "cooperating fully" -in the usual tradition of the digital underground. - -He gave Foley a complete run of Phrack, printed out in a set -of three-ring binders. He handed over his electronic mailing list -of Phrack subscribers. Knight Lightning was grilled for four hours -by Foley and his cohorts. Knight Lightning admitted that Prophet -had passed him the E911 Document, and he admitted that he had known -it was stolen booty from a hacker raid on a telephone company. -Knight Lightning signed a statement to this effect, and agreed, -in writing, to cooperate with investigators. - -Next day--January 19, 1990, a Friday --the Secret Service returned -with a search warrant, and thoroughly searched Knight Lightning's -upstairs room in the fraternity house. They took all his floppy disks, -though, interestingly, they left Knight Lightning in possession -of both his computer and his modem. (The computer had no hard disk, -and in Foley's judgement was not a store of evidence.) But this was a -very minor bright spot among Knight Lightning's rapidly multiplying troubles. -By this time, Knight Lightning was in plenty of hot water, not only with -federal police, prosecutors, telco investigators, and university security, -but with the elders of his own campus fraternity, who were outraged -to think that they had been unwittingly harboring a federal computer-criminal. - -On Monday, Knight Lightning was summoned to Chicago, where he was -further grilled by Foley and USSS veteran agent Barbara Golden, this time -with an attorney present. And on Tuesday, he was formally indicted -by a federal grand jury. - -The trial of Knight Lightning, which occurred on July 24-27, 1990, -was the crucial show-trial of the Hacker Crackdown. We will examine -the trial at some length in Part Four of this book. - -In the meantime, we must continue our dogged pursuit of the E911 Document. - -It must have been clear by January 1990 that the E911 Document, -in the form Phrack had published it back in February 1989, -had gone off at the speed of light in at least a hundred -and fifty different directions. To attempt to put this -electronic genie back in the bottle was flatly impossible. - -And yet, the E911 Document was STILL stolen property, -formally and legally speaking. Any electronic transference -of this document, by anyone unauthorized to have it, -could be interpreted as an act of wire fraud. Interstate -transfer of stolen property, including electronic property, -was a federal crime. - -The Chicago Computer Fraud and Abuse Task Force had been assured -that the E911 Document was worth a hefty sum of money. In fact, -they had a precise estimate of its worth from BellSouth security personnel: -$79,449. A sum of this scale seemed to warrant vigorous prosecution. -Even if the damage could not be undone, at least this large sum -offered a good legal pretext for stern punishment of the thieves. -It seemed likely to impress judges and juries. And it could be used -in court to mop up the Legion of Doom. - -The Atlanta crowd was already in the bag, by the time -the Chicago Task Force had gotten around to Phrack. -But the Legion was a hydra-headed thing. In late 89, -a brand-new Legion of Doom board, "Phoenix Project," -had gone up in Austin, Texas. Phoenix Project was sysoped -by no less a man than the Mentor himself, ably assisted by -University of Texas student and hardened Doomster "Erik Bloodaxe." - -As we have seen from his Phrack manifesto, the Mentor was a hacker -zealot who regarded computer intrusion as something close to a moral duty. -Phoenix Project was an ambitious effort, intended to revive the digital -underground to what Mentor considered the full flower of the early 80s. -The Phoenix board would also boldly bring elite hackers face-to-face -with the telco "opposition." On "Phoenix," America's cleverest hackers -would supposedly shame the telco squareheads out of their stick-in-the-mud -attitudes, and perhaps convince them that the Legion of Doom elite were really -an all-right crew. The premiere of "Phoenix Project" was heavily trumpeted -by Phrack,and "Phoenix Project" carried a complete run of Phrack issues, -including the E911 Document as Phrack had published it. - -Phoenix Project was only one of many--possibly hundreds--of nodes and boards -all over America that were in guilty possession of the E911 Document. -But Phoenix was an outright, unashamed Legion of Doom board. -Under Mentor's guidance, it was flaunting itself in the face -of telco security personnel. Worse yet, it was actively trying -to WIN THEM OVER as sympathizers for the digital underground elite. -"Phoenix" had no cards or codes on it. Its hacker elite considered -Phoenix at least technically legal. But Phoenix was a corrupting influence, -where hacker anarchy was eating away like digital acid at the underbelly -of corporate propriety. - -The Chicago Computer Fraud and Abuse Task Force now prepared -to descend upon Austin, Texas. - -Oddly, not one but TWO trails of the Task Force's investigation led -toward Austin. The city of Austin, like Atlanta, had made itself -a bulwark of the Sunbelt's Information Age, with a strong university -research presence, and a number of cutting-edge electronics companies, -including Motorola, Dell, CompuAdd, IBM, Sematech and MCC. - -Where computing machinery went, hackers generally followed. -Austin boasted not only "Phoenix Project," currently LoD's -most flagrant underground board, but a number of UNIX nodes. - -One of these nodes was "Elephant," run by a UNIX consultant -named Robert Izenberg. Izenberg, in search of a relaxed Southern -lifestyle and a lowered cost-of-living, had recently migrated -to Austin from New Jersey. In New Jersey, Izenberg had worked -for an independent contracting company, programming UNIX code for -AT&T itself. "Terminus" had been a frequent user on Izenberg's -privately owned Elephant node. - -Having interviewed Terminus and examined the records on Netsys, -the Chicago Task Force were now convinced that they had discovered -an underground gang of UNIX software pirates, who were demonstrably -guilty of interstate trafficking in illicitly copied AT&T source code. -Izenberg was swept into the dragnet around Terminus, the self-proclaimed -ultimate UNIX hacker. - -Izenberg, in Austin, had settled down into a UNIX job -with a Texan branch of IBM. Izenberg was no longer -working as a contractor for AT&T, but he had friends -in New Jersey, and he still logged on to AT&T UNIX -computers back in New Jersey, more or less whenever -it pleased him. Izenberg's activities appeared highly -suspicious to the Task Force. Izenberg might well be -breaking into AT&T computers, swiping AT&T software, -and passing it to Terminus and other possible confederates, -through the UNIX node network. And this data was worth, -not merely $79,499, but hundreds of thousands of dollars! - -On February 21, 1990, Robert Izenberg arrived home -from work at IBM to find that all the computers -had mysteriously vanished from his Austin apartment. -Naturally he assumed that he had been robbed. -His "Elephant" node, his other machines, his notebooks, -his disks, his tapes, all gone! However, nothing much -else seemed disturbed--the place had not been ransacked. -The puzzle becaming much stranger some five minutes later. -Austin U. S. Secret Service Agent Al Soliz, accompanied by -University of Texas campus-security officer Larry Coutorie -and the ubiquitous Tim Foley, made their appearance at Izenberg's door. -They were in plain clothes: slacks, polo shirts. They came in, -and Tim Foley accused Izenberg of belonging to the Legion of Doom. - -Izenberg told them that he had never heard of the "Legion of Doom." -And what about a certain stolen E911 Document, that posed a direct -threat to the police emergency lines? Izenberg claimed that he'd -never heard of that, either. - -His interrogators found this difficult to believe. -Didn't he know Terminus? - -Who? - -They gave him Terminus's real name. Oh yes, said Izenberg. -He knew THAT guy all right--he was leading discussions -on the Internet about AT&T computers, especially the AT&T 3B2. - -AT&T had thrust this machine into the marketplace, -but, like many of AT&T's ambitious attempts to enter -the computing arena, the 3B2 project had something less -than a glittering success. Izenberg himself had been -a contractor for the division of AT&T that supported the 3B2. -The entire division had been shut down. - -Nowadays, the cheapest and quickest way to get help with this -fractious piece of machinery was to join one of Terminus's -discussion groups on the Internet, where friendly and knowledgeable -hackers would help you for free. Naturally the remarks within this -group were less than flattering about the Death Star. . .was -THAT the problem? - -Foley told Izenberg that Terminus had been acquiring hot software -through his, Izenberg's, machine. - -Izenberg shrugged this off. A good eight megabytes of data flowed -through his UUCP site every day. UUCP nodes spewed data like fire hoses. -Elephant had been directly linked to Netsys--not surprising, since Terminus -was a 3B2 expert and Izenberg had been a 3B2 contractor. -Izenberg was also linked to "attctc" and the University of Texas. -Terminus was a well-known UNIX expert, and might have been up to -all manner of hijinks on Elephant. Nothing Izenberg could do about that. -That was physically impossible. Needle in a haystack. - -In a four-hour grilling, Foley urged Izenberg to come clean -and admit that he was in conspiracy with Terminus, -and a member of the Legion of Doom. - -Izenberg denied this. He was no weirdo teenage hacker-- -he was thirty-two years old, and didn't even have a "handle." -Izenberg was a former TV technician and electronics specialist -who had drifted into UNIX consulting as a full-grown adult. -Izenberg had never met Terminus, physically. He'd once bought -a cheap high-speed modem from him, though. - -Foley told him that this modem (a Telenet T2500 which ran at 19.2 kilobaud, -and which had just gone out Izenberg's door in Secret Service custody) -was likely hot property. Izenberg was taken aback to hear this; but then -again, most of Izenberg's equipment, like that of most freelance professionals -in the industry, was discounted, passed hand-to-hand through various kinds -of barter and gray-market. There was no proof that the modem was stolen, -and even if it were, Izenberg hardly saw how that gave them the right -to take every electronic item in his house. - -Still, if the United States Secret Service figured they needed -his computer for national security reasons--or whatever-- -then Izenberg would not kick. He figured he would somehow -make the sacrifice of his twenty thousand dollars' worth -of professional equipment, in the spirit of full cooperation -and good citizenship. - -Robert Izenberg was not arrested. Izenberg was not charged with any crime. -His UUCP node--full of some 140 megabytes of the files, mail, and data -of himself and his dozen or so entirely innocent users--went out the door -as "evidence." Along with the disks and tapes, Izenberg had lost about -800 megabytes of data. - -Six months would pass before Izenberg decided to phone the Secret Service -and ask how the case was going. That was the first time that Robert Izenberg -would ever hear the name of William Cook. As of January 1992, a full -two years after the seizure, Izenberg, still not charged with any crime, -would be struggling through the morass of the courts, in hope of recovering -his thousands of dollars' worth of seized equipment. - -In the meantime, the Izenberg case received absolutely no press coverage. -The Secret Service had walked into an Austin home, removed a UNIX bulletin- -board system, and met with no operational difficulties whatsoever. - -Except that word of a crackdown had percolated through the Legion of Doom. -"The Mentor" voluntarily shut down "The Phoenix Project." It seemed a pity, -especially as telco security employees had, in fact, shown up on Phoenix, -just as he had hoped--along with the usual motley crowd of LoD heavies, -hangers-on, phreaks, hackers and wannabes. There was "Sandy" Sandquist from -US SPRINT security, and some guy named Henry Kluepfel, from Bellcore itself! -Kluepfel had been trading friendly banter with hackers on Phoenix since -January 30th (two weeks after the Martin Luther King Day Crash). -The presence of such a stellar telco official seemed quite the coup -for Phoenix Project. - -Still, Mentor could judge the climate. Atlanta in ruins, -Phrack in deep trouble, something weird going on with UNIX nodes-- -discretion was advisable. Phoenix Project went off-line. - -Kluepfel, of course, had been monitoring this LoD bulletin -board for his own purposes--and those of the Chicago unit. -As far back as June 1987, Kluepfel had logged on to a Texas -underground board called "Phreak Klass 2600." There he'd -discovered an Chicago youngster named "Shadowhawk," -strutting and boasting about rifling AT&T computer files, -and bragging of his ambitions to riddle AT&T's Bellcore -computers with trojan horse programs. Kluepfel had passed -the news to Cook in Chicago, Shadowhawk's computers -had gone out the door in Secret Service custody, -and Shadowhawk himself had gone to jail. - -Now it was Phoenix Project's turn. Phoenix Project postured -about "legality" and "merely intellectual interest," but it reeked -of the underground. It had Phrack on it. It had the E911 Document. -It had a lot of dicey talk about breaking into systems, including some -bold and reckless stuff about a supposed "decryption service" that Mentor -and friends were planning to run, to help crack encrypted passwords off -of hacked systems. - -Mentor was an adult. There was a bulletin board at his place of work, -as well. Kleupfel logged onto this board, too, and discovered it to be -called "Illuminati." It was run by some company called Steve Jackson Games. - -On March 1, 1990, the Austin crackdown went into high gear. - -On the morning of March 1--a Thursday--21-year-old University of Texas -student "Erik Bloodaxe," co-sysop of Phoenix Project and an avowed member -of the Legion of Doom, was wakened by a police revolver levelled at his head. - -Bloodaxe watched, jittery, as Secret Service agents -appropriated his 300 baud terminal and, rifling his files, -discovered his treasured source-code for Robert Morris's -notorious Internet Worm. But Bloodaxe, a wily operator, -had suspected that something of the like might be coming. -All his best equipment had been hidden away elsewhere. -The raiders took everything electronic, however, -including his telephone. They were stymied by his -hefty arcade-style Pac-Man game, and left it in place, -as it was simply too heavy to move. - -Bloodaxe was not arrested. He was not charged with any crime. -A good two years later, the police still had what they had -taken from him, however. - -The Mentor was less wary. The dawn raid rousted him and his wife -from bed in their underwear, and six Secret Service agents, -accompanied by an Austin policeman and Henry Kluepfel himself, -made a rich haul. Off went the works, into the agents' white -Chevrolet minivan: an IBM PC-AT clone with 4 meg of RAM and -a 120-meg hard disk; a Hewlett-Packard LaserJet II printer; -a completely legitimate and highly expensive SCO-Xenix 286 -operating system; Pagemaker disks and documentation; -and the Microsoft Word word-processing program. Mentor's wife -had her incomplete academic thesis stored on the hard-disk; -that went, too, and so did the couple's telephone. As of two years later, -all this property remained in police custody. - -Mentor remained under guard in his apartment as agents prepared -to raid Steve Jackson Games. The fact that this was a business -headquarters and not a private residence did not deter the agents. -It was still very early; no one was at work yet. The agents prepared -to break down the door, but Mentor, eavesdropping on the Secret Service -walkie-talkie traffic, begged them not to do it, and offered his key -to the building. - -The exact details of the next events are unclear. The agents -would not let anyone else into the building. Their search warrant, -when produced, was unsigned. Apparently they breakfasted from the local -"Whataburger," as the litter from hamburgers was later found inside. -They also extensively sampled a bag of jellybeans kept by an SJG employee. -Someone tore a "Dukakis for President" sticker from the wall. - -SJG employees, diligently showing up for the day's work, were met -at the door and briefly questioned by U.S. Secret Service agents. -The employees watched in astonishment as agents wielding crowbars -and screwdrivers emerged with captive machines. They attacked -outdoor storage units with boltcutters. The agents wore -blue nylon windbreakers with "SECRET SERVICE" stencilled -across the back, with running-shoes and jeans. - -Jackson's company lost three computers, several hard-disks, -hundred of floppy disks, two monitors, three modems, -a laser printer, various powercords, cables, and adapters -(and, oddly, a small bag of screws, bolts and nuts). -The seizure of Illuminati BBS deprived SJG of all the programs, -text files, and private e-mail on the board. The loss of two other -SJG computers was a severe blow as well, since it caused the loss -of electronically stored contracts, financial projections, -address directories, mailing lists, personnel files, -business correspondence, and, not least, the drafts -of forthcoming games and gaming books. - -No one at Steve Jackson Games was arrested. No one was accused -of any crime. No charges were filed. Everything appropriated -was officially kept as "evidence" of crimes never specified. - -After the Phrack show-trial, the Steve Jackson Games scandal -was the most bizarre and aggravating incident of the Hacker -Crackdown of 1990. This raid by the Chicago Task Force -on a science-fiction gaming publisher was to rouse a -swarming host of civil liberties issues, and gave rise -to an enduring controversy that was still re-complicating itself, -and growing in the scope of its implications, a full two years later. - -The pursuit of the E911 Document stopped with the Steve Jackson Games raid. -As we have seen, there were hundreds, perhaps thousands of computer users -in America with the E911 Document in their possession. Theoretically, -Chicago had a perfect legal right to raid any of these people, -and could have legally seized the machines of anybody who subscribed to Phrack. -However, there was no copy of the E911 Document on Jackson's Illuminati board. -And there the Chicago raiders stopped dead; they have not raided anyone since. - -It might be assumed that Rich Andrews and Charlie Boykin, who had brought -the E911 Document to the attention of telco security, might be spared -any official suspicion. But as we have seen, the willingness to -"cooperate fully" offers little, if any, assurance against federal -anti-hacker prosecution. - -Richard Andrews found himself in deep trouble, thanks to the E911 Document. -Andrews lived in Illinois, the native stomping grounds of the Chicago -Task Force. On February 3 and 6, both his home and his place of work -were raided by USSS. His machines went out the door, too, and he was -grilled at length (though not arrested). Andrews proved to be in -purportedly guilty possession of: UNIX SVR 3.2; UNIX SVR 3.1; UUCP; -PMON; WWB; IWB; DWB; NROFF; KORN SHELL '88; C++; and QUEST, -among other items. Andrews had received this proprietary code-- -which AT&T officially valued at well over $250,000--through the -UNIX network, much of it supplied to him as a personal favor by Terminus. -Perhaps worse yet, Andrews admitted to returning the favor, by passing -Terminus a copy of AT&T proprietary STARLAN source code. - -Even Charles Boykin, himself an AT&T employee, entered some very hot water. -By 1990, he'd almost forgotten about the E911 problem he'd reported in -September 88; in fact, since that date, he'd passed two more security alerts -to Jerry Dalton, concerning matters that Boykin considered far worse than -the E911 Document. - -But by 1990, year of the crackdown, AT&T Corporate Information Security -was fed up with "Killer." This machine offered no direct income to AT&T, -and was providing aid and comfort to a cloud of suspicious yokels -from outside the company, some of them actively malicious toward AT&T, -its property, and its corporate interests. Whatever goodwill and publicity -had been won among Killer's 1,500 devoted users was considered no longer -worth the security risk. On February 20, 1990, Jerry Dalton arrived in -Dallas and simply unplugged the phone jacks, to the puzzled alarm -of Killer's many Texan users. Killer went permanently off-line, -with the loss of vast archives of programs and huge quantities -of electronic mail; it was never restored to service. AT&T showed -no particular regard for the "property" of these 1,500 people. -Whatever "property" the users had been storing on AT&T's computer -simply vanished completely. - -Boykin, who had himself reported the E911 problem, -now found himself under a cloud of suspicion. In a weird -private-security replay of the Secret Service seizures, -Boykin's own home was visited by AT&T Security and his -own machines were carried out the door. - -However, there were marked special features in the Boykin case. -Boykin's disks and his personal computers were swiftly examined -by his corporate employers and returned politely in just two days-- -(unlike Secret Service seizures, which commonly take months or years). -Boykin was not charged with any crime or wrongdoing, and he kept his job -with AT&T (though he did retire from AT&T in September 1991, -at the age of 52). - -It's interesting to note that the US Secret Service somehow failed -to seize Boykin's "Killer" node and carry AT&T's own computer out the door. -Nor did they raid Boykin's home. They seemed perfectly willing to take the -word of AT&T Security that AT&T's employee, and AT&T's "Killer" node, -were free of hacker contraband and on the up-and-up. - -It's digital water-under-the-bridge at this point, as Killer's -3,200 megabytes of Texan electronic community were erased in 1990, -and "Killer" itself was shipped out of the state. - -But the experiences of Andrews and Boykin, and the users of their systems, -remained side issues. They did not begin to assume the social, political, -and legal importance that gathered, slowly but inexorably, around the issue -of the raid on Steve Jackson Games. - -# - -We must now turn our attention to Steve Jackson Games itself, -and explain what SJG was, what it really did, and how it had -managed to attract this particularly odd and virulent kind of trouble. -The reader may recall that this is not the first but the second time -that the company has appeared in this narrative; a Steve Jackson game -called GURPS was a favorite pastime of Atlanta hacker Urvile, -and Urvile's science-fictional gaming notes had been mixed up -promiscuously with notes about his actual computer intrusions. - -First, Steve Jackson Games, Inc., was NOT a publisher of "computer games." -SJG published "simulation games," parlor games that were played on paper, -with pencils, and dice, and printed guidebooks full of rules and -statistics tables. There were no computers involved in the games themselves. -When you bought a Steve Jackson Game, you did not receive any software disks. -What you got was a plastic bag with some cardboard game tokens, -maybe a few maps or a deck of cards. Most of their products were books. - -However, computers WERE deeply involved in the Steve Jackson Games business. -Like almost all modern publishers, Steve Jackson and his fifteen employees -used computers to write text, to keep accounts, and to run the business -generally. They also used a computer to run their official bulletin board -system for Steve Jackson Games, a board called Illuminati. On Illuminati, -simulation gamers who happened to own computers and modems could associate, -trade mail, debate the theory and practice of gaming, and keep up with the -company's news and its product announcements. - -Illuminati was a modestly popular board, run on a small computer -with limited storage, only one phone-line, and no ties to large-scale -computer networks. It did, however, have hundreds of users, -many of them dedicated gamers willing to call from out-of-state. - -Illuminati was NOT an "underground" board. It did not feature hints -on computer intrusion, or "anarchy files," or illicitly posted -credit card numbers, or long-distance access codes. -Some of Illuminati's users, however, were members of the Legion of Doom. -And so was one of Steve Jackson's senior employees--the Mentor. -The Mentor wrote for Phrack, and also ran an underground board, -Phoenix Project--but the Mentor was not a computer professional. -The Mentor was the managing editor of Steve Jackson Games and -a professional game designer by trade. These LoD members did not -use Illuminati to help their HACKING activities. They used it to -help their GAME-PLAYING activities--and they were even more dedicated -to simulation gaming than they were to hacking. - -"Illuminati" got its name from a card-game that Steve Jackson himself, -the company's founder and sole owner, had invented. This multi-player -card-game was one of Mr Jackson's best-known, most successful, -most technically innovative products. "Illuminati" was a game -of paranoiac conspiracy in which various antisocial cults warred -covertly to dominate the world. "Illuminati" was hilarious, -and great fun to play, involving flying saucers, the CIA, the KGB, -the phone companies, the Ku Klux Klan, the South American Nazis, -the cocaine cartels, the Boy Scouts, and dozens of other splinter groups -from the twisted depths of Mr. Jackson's professionally fervid imagination. -For the uninitiated, any public discussion of the "Illuminati" card-game -sounded, by turns, utterly menacing or completely insane. - -And then there was SJG's "Car Wars," in which souped-up armored hot-rods -with rocket-launchers and heavy machine-guns did battle on the American -highways of the future. The lively Car Wars discussion on the Illuminati -board featured many meticulous, painstaking discussions of the effects -of grenades, land-mines, flamethrowers and napalm. It sounded like -hacker anarchy files run amuck. - -Mr Jackson and his co-workers earned their daily bread by supplying people -with make-believe adventures and weird ideas. The more far-out, the better. - -Simulation gaming is an unusual pastime, but gamers have not -generally had to beg the permission of the Secret Service to exist. -Wargames and role-playing adventures are an old and honored pastime, -much favored by professional military strategists. Once little-known, -these games are now played by hundreds of thousands of enthusiasts -throughout North America, Europe and Japan. Gaming-books, once restricted -to hobby outlets, now commonly appear in chain-stores like B. Dalton's -and Waldenbooks, and sell vigorously. - -Steve Jackson Games, Inc., of Austin, Texas, was a games company -of the middle rank. In 1989, SJG grossed about a million dollars. -Jackson himself had a good reputation in his industry as a talented -and innovative designer of rather unconventional games, but his company -was something less than a titan of the field--certainly not like the -multimillion-dollar TSR Inc., or Britain's gigantic "Games Workshop." -SJG's Austin headquarters was a modest two-story brick office-suite, -cluttered with phones, photocopiers, fax machines and computers. -It bustled with semi-organized activity and was littered with -glossy promotional brochures and dog-eared science-fiction novels. -Attached to the offices was a large tin-roofed warehouse piled twenty feet -high with cardboard boxes of games and books. Despite the weird imaginings -that went on within it, the SJG headquarters was quite a quotidian, -everyday sort of place. It looked like what it was: a publishers' digs. - -Both "Car Wars" and "Illuminati" were well-known, popular games. -But the mainstay of the Jackson organization was their Generic Universal -Role-Playing System, "G.U.R.P.S." The GURPS system was considered solid -and well-designed, an asset for players. But perhaps the most popular -feature of the GURPS system was that it allowed gaming-masters to design -scenarios that closely resembled well-known books, movies, and other works -of fantasy. Jackson had licensed and adapted works from many science fiction -and fantasy authors. There was GURPS Conan, GURPS Riverworld, -GURPS Horseclans, GURPS Witch World, names eminently familiar -to science-fiction readers. And there was GURPS Special Ops, -from the world of espionage fantasy and unconventional warfare. - -And then there was GURPS Cyberpunk. - -"Cyberpunk" was a term given to certain science fiction writers -who had entered the genre in the 1980s. "Cyberpunk," as the label implies, -had two general distinguishing features. First, its writers had a compelling -interest in information technology, an interest closely akin -to science fiction's earlier fascination with space travel. -And second, these writers were "punks," with all the -distinguishing features that that implies: Bohemian artiness, -youth run wild, an air of deliberate rebellion, funny clothes and hair, -odd politics, a fondness for abrasive rock and roll; in a word, trouble. - -The "cyberpunk" SF writers were a small group of mostly college-educated -white middle-class litterateurs, scattered through the US and Canada. -Only one, Rudy Rucker, a professor of computer science in Silicon Valley, -could rank with even the humblest computer hacker. But, except for -Professor Rucker, the "cyberpunk" authors were not programmers -or hardware experts; they considered themselves artists -(as, indeed, did Professor Rucker). However, these writers -all owned computers, and took an intense and public interest -in the social ramifications of the information industry. - -The cyberpunks had a strong following among the global generation -that had grown up in a world of computers, multinational networks, -and cable television. Their outlook was considered somewhat morbid, -cynical, and dark, but then again, so was the outlook of their -generational peers. As that generation matured and increased -in strength and influence, so did the cyberpunks. -As science-fiction writers went, they were doing -fairly well for themselves. By the late 1980s, -their work had attracted attention from gaming companies, -including Steve Jackson Games, which was planning a cyberpunk -simulation for the flourishing GURPS gaming-system. - -The time seemed ripe for such a product, which had already been proven -in the marketplace. The first games- company out of the gate, -with a product boldly called "Cyberpunk" in defiance of possible -infringement-of-copyright suits, had been an upstart group called -R. Talsorian. Talsorian's Cyberpunk was a fairly decent game, -but the mechanics of the simulation system left a lot to be desired. -Commercially, however, the game did very well. - -The next cyberpunk game had been the even more successful Shadowrun -by FASA Corporation. The mechanics of this game were fine, but the -scenario was rendered moronic by sappy fantasy elements like elves, -trolls, wizards, and dragons--all highly ideologically-incorrect, -according to the hard-edged, high-tech standards of cyberpunk science fiction. - -Other game designers were champing at the bit. Prominent among them -was the Mentor, a gentleman who, like most of his friends in the -Legion of Doom, was quite the cyberpunk devotee. Mentor reasoned -that the time had come for a REAL cyberpunk gaming-book--one that the -princes of computer-mischief in the Legion of Doom could play without -laughing themselves sick. This book, GURPS Cyberpunk, would reek -of culturally on-line authenticity. - -Mentor was particularly well-qualified for this task. -Naturally, he knew far more about computer-intrusion -and digital skullduggery than any previously published -cyberpunk author. Not only that, but he was good at his work. -A vivid imagination, combined with an instinctive feeling -for the working of systems and, especially, the loopholes -within them, are excellent qualities for a professional game designer. - -By March 1st, GURPS Cyberpunk was almost complete, ready to print and ship. -Steve Jackson expected vigorous sales for this item, which, he hoped, -would keep the company financially afloat for several months. -GURPS Cyberpunk, like the other GURPS "modules," was not a "game" -like a Monopoly set, but a BOOK: a bound paperback book the size -of a glossy magazine, with a slick color cover, and pages full of text, -illustrations, tables and footnotes. It was advertised as a game, -and was used as an aid to game-playing, but it was a book, -with an ISBN number, published in Texas, copyrighted, -and sold in bookstores. - -And now, that book, stored on a computer, had gone out the door -in the custody of the Secret Service. - -The day after the raid, Steve Jackson visited the local Secret Service -headquarters with a lawyer in tow. There he confronted Tim Foley -(still in Austin at that time) and demanded his book back. But there -was trouble. GURPS Cyberpunk, alleged a Secret Service agent to astonished -businessman Steve Jackson, was "a manual for computer crime." - -"It's science fiction," Jackson said. - -"No, this is real." - -This statement was repeated several times, by several agents. -Jackson's ominously accurate game had passed from pure, -obscure, small-scale fantasy into the impure, highly publicized, -large-scale fantasy of the Hacker Crackdown. - -No mention was made of the real reason for the search. -According to their search warrant, the raiders had expected -to find the E911 Document stored on Jackson's bulletin board system. -But that warrant was sealed; a procedure that most law enforcement agencies -will use only when lives are demonstrably in danger. The raiders' -true motives were not discovered until the Jackson search-warrant -was unsealed by his lawyers, many months later. The Secret Service, -and the Chicago Computer Fraud and Abuse Task Force, -said absolutely nothing to Steve Jackson about any threat -to the police 911 System. They said nothing about the Atlanta Three, -nothing about Phrack or Knight Lightning, nothing about Terminus. - -Jackson was left to believe that his computers had been seized because -he intended to publish a science fiction book that law enforcement -considered too dangerous to see print. - -This misconception was repeated again and again, for months, -to an ever-widening public audience. It was not the truth of the case; -but as months passed, and this misconception was publicly printed again -and again, it became one of the few publicly known "facts" about -the mysterious Hacker Crackdown. The Secret Service had seized a computer -to stop the publication of a cyberpunk science fiction book. - -The second section of this book, "The Digital Underground," -is almost finished now. We have become acquainted with all -the major figures of this case who actually belong to the -underground milieu of computer intrusion. We have some idea -of their history, their motives, their general modus operandi. -We now know, I hope, who they are, where they came from, -and more or less what they want. In the next section of this book, -"Law and Order," we will leave this milieu and directly enter the -world of America's computer-crime police. - -At this point, however, I have another figure to introduce: myself. - -My name is Bruce Sterling. I live in Austin, Texas, where I am -a science fiction writer by trade: specifically, a CYBERPUNK -science fiction writer. - -Like my "cyberpunk" colleagues in the U.S. and Canada, -I've never been entirely happy with this literary label-- -especially after it became a synonym for computer criminal. -But I did once edit a book of stories by my colleagues, -called Mirrorshades: the Cyberpunk Anthology, and I've -long been a writer of literary-critical cyberpunk manifestos. -I am not a "hacker" of any description, though I do have readers -in the digital underground. - -When the Steve Jackson Games seizure occurred, I naturally took -an intense interest. If "cyberpunk" books were being banned -by federal police in my own home town, I reasonably wondered -whether I myself might be next. Would my computer be seized -by the Secret Service? At the time, I was in possession -of an aging Apple IIe without so much as a hard disk. -If I were to be raided as an author of computer-crime manuals, -the loss of my feeble word-processor would likely provoke more -snickers than sympathy. - -I'd known Steve Jackson for many years. We knew -one another as colleagues, for we frequented -the same local science-fiction conventions. -I'd played Jackson games, and recognized his cleverness; -but he certainly had never struck me as a potential mastermind -of computer crime. - -I also knew a little about computer bulletin-board systems. -In the mid-1980s I had taken an active role in an Austin board -called "SMOF-BBS," one of the first boards dedicated to science fiction. -I had a modem, and on occasion I'd logged on to Illuminati, -which always looked entertainly wacky, but certainly harmless enough. - -At the time of the Jackson seizure, I had no experience -whatsoever with underground boards. But I knew that no one -on Illuminati talked about breaking into systems illegally, -or about robbing phone companies. Illuminati didn't even -offer pirated computer games. Steve Jackson, like many creative artists, -was markedly touchy about theft of intellectual property. - -It seemed to me that Jackson was either seriously suspected -of some crime--in which case, he would be charged soon, -and would have his day in court--or else he was innocent, -in which case the Secret Service would quickly return his equipment, -and everyone would have a good laugh. I rather expected the good laugh. -The situation was not without its comic side. The raid, known -as the "Cyberpunk Bust" in the science fiction community, -was winning a great deal of free national publicity both -for Jackson himself and the "cyberpunk" science fiction -writers generally. - -Besides, science fiction people are used to being misinterpreted. -Science fiction is a colorful, disreputable, slipshod occupation, -full of unlikely oddballs, which, of course, is why we like it. -Weirdness can be an occupational hazard in our field. People who -wear Halloween costumes are sometimes mistaken for monsters. - -Once upon a time--back in 1939, in New York City-- -science fiction and the U.S. Secret Service collided in -a comic case of mistaken identity. This weird incident -involved a literary group quite famous in science fiction, -known as "the Futurians," whose membership included -such future genre greats as Isaac Asimov, Frederik Pohl, -and Damon Knight. The Futurians were every bit as -offbeat and wacky as any of their spiritual descendants, -including the cyberpunks, and were given to communal living, -spontaneous group renditions of light opera, and midnight fencing -exhibitions on the lawn. The Futurians didn't have bulletin -board systems, but they did have the technological equivalent -in 1939--mimeographs and a private printing press. These were -in steady use, producing a stream of science-fiction fan magazines, -literary manifestos, and weird articles, which were picked up -in ink-sticky bundles by a succession of strange, gangly, -spotty young men in fedoras and overcoats. - -The neighbors grew alarmed at the antics of the Futurians -and reported them to the Secret Service as suspected counterfeiters. -In the winter of 1939, a squad of USSS agents with drawn guns burst into -"Futurian House," prepared to confiscate the forged currency and illicit -printing presses. There they discovered a slumbering science fiction fan -named George Hahn, a guest of the Futurian commune who had just arrived -in New York. George Hahn managed to explain himself and his group, -and the Secret Service agents left the Futurians in peace henceforth. -(Alas, Hahn died in 1991, just before I had discovered this astonishing -historical parallel, and just before I could interview him for this book.) - -But the Jackson case did not come to a swift and comic end. -No quick answers came his way, or mine; no swift reassurances -that all was right in the digital world, that matters were well -in hand after all. Quite the opposite. In my alternate role -as a sometime pop-science journalist, I interviewed Jackson -and his staff for an article in a British magazine. -The strange details of the raid left me more concerned than ever. -Without its computers, the company had been financially -and operationally crippled. Half the SJG workforce, -a group of entirely innocent people, had been sorrowfully fired, -deprived of their livelihoods by the seizure. It began to dawn on me -that authors--American writers--might well have their computers seized, -under sealed warrants, without any criminal charge; and that, -as Steve Jackson had discovered, there was no immediate recourse for this. -This was no joke; this wasn't science fiction; this was real. - -I determined to put science fiction aside until I had discovered -what had happened and where this trouble had come from. -It was time to enter the purportedly real world of electronic -free expression and computer crime. Hence, this book. -Hence, the world of the telcos; and the world of the digital underground; -and next, the world of the police. - - - -PART THREE: LAW AND ORDER - - -Of the various anti-hacker activities of 1990, "Operation Sundevil" -had by far the highest public profile. The sweeping, nationwide -computer seizures of May 8, 1990 were unprecedented in scope and highly, -if rather selectively, publicized. - -Unlike the efforts of the Chicago Computer Fraud and Abuse Task Force, -"Operation Sundevil" was not intended to combat "hacking" in the sense -of computer intrusion or sophisticated raids on telco switching stations. -Nor did it have anything to do with hacker misdeeds with AT&T's software, -or with Southern Bell's proprietary documents. - -Instead, "Operation Sundevil" was a crackdown on those traditional scourges -of the digital underground: credit-card theft and telephone code abuse. -The ambitious activities out of Chicago, and the somewhat lesser-known -but vigorous anti-hacker actions of the New York State Police in 1990, -were never a part of "Operation Sundevil" per se, which was based in Arizona. - -Nevertheless, after the spectacular May 8 raids, the public, misled by -police secrecy, hacker panic, and a puzzled national press-corps, -conflated all aspects of the nationwide crackdown in 1990 under -the blanket term "Operation Sundevil." "Sundevil" is still the best-known -synonym for the crackdown of 1990. But the Arizona organizers of "Sundevil" -did not really deserve this reputation--any more, for instance, than all -hackers deserve a reputation as "hackers." - -There was some justice in this confused perception, though. -For one thing, the confusion was abetted by the Washington office -of the Secret Service, who responded to Freedom of Information Act -requests on "Operation Sundevil" by referring investigators -to the publicly known cases of Knight Lightning and the Atlanta Three. -And "Sundevil" was certainly the largest aspect of the Crackdown, -the most deliberate and the best-organized. As a crackdown on electronic -fraud, "Sundevil" lacked the frantic pace of the war on the Legion of Doom; -on the contrary, Sundevil's targets were picked out with cool deliberation -over an elaborate investigation lasting two full years. - -And once again the targets were bulletin board systems. - -Boards can be powerful aids to organized fraud. Underground boards carry -lively, extensive, detailed, and often quite flagrant "discussions" of -lawbreaking techniques and lawbreaking activities. "Discussing" crime -in the abstract, or "discussing" the particulars of criminal cases, -is not illegal--but there are stern state and federal laws against -coldbloodedly conspiring in groups in order to commit crimes. - -In the eyes of police, people who actively conspire to break the law -are not regarded as "clubs," "debating salons," "users' groups," or -"free speech advocates." Rather, such people tend to find themselves -formally indicted by prosecutors as "gangs," "racketeers," "corrupt -organizations" and "organized crime figures." - -What's more, the illicit data contained on outlaw boards goes well beyond -mere acts of speech and/or possible criminal conspiracy. As we have seen, -it was common practice in the digital underground to post purloined telephone -codes on boards, for any phreak or hacker who cared to abuse them. Is posting -digital booty of this sort supposed to be protected by the First Amendment? -Hardly--though the issue, like most issues in cyberspace, is not entirely -resolved. Some theorists argue that to merely RECITE a number publicly -is not illegal--only its USE is illegal. But anti-hacker police point out -that magazines and newspapers (more traditional forms of free expression) -never publish stolen telephone codes (even though this might well -raise their circulation). - -Stolen credit card numbers, being riskier and more valuable, -were less often publicly posted on boards--but there is no question -that some underground boards carried "carding" traffic, -generally exchanged through private mail. - -Underground boards also carried handy programs for "scanning" telephone -codes and raiding credit card companies, as well as the usual obnoxious -galaxy of pirated software, cracked passwords, blue-box schematics, -intrusion manuals, anarchy files, porn files, and so forth. - -But besides their nuisance potential for the spread of illicit knowledge, -bulletin boards have another vitally interesting aspect for the -professional investigator. Bulletin boards are cram-full of EVIDENCE. -All that busy trading of electronic mail, all those hacker boasts, -brags and struts, even the stolen codes and cards, can be neat, -electronic, real-time recordings of criminal activity. -As an investigator, when you seize a pirate board, you have -scored a coup as effective as tapping phones or intercepting mail. -However, you have not actually tapped a phone or intercepted a letter. -The rules of evidence regarding phone-taps and mail interceptions are old, -stern and well-understood by police, prosecutors and defense attorneys alike. -The rules of evidence regarding boards are new, waffling, and understood -by nobody at all. - -Sundevil was the largest crackdown on boards in world history. -On May 7, 8, and 9, 1990, about forty-two computer systems were seized. -Of those forty-two computers, about twenty-five actually were running boards. -(The vagueness of this estimate is attributable to the vagueness of -(a) what a "computer system" is, and (b) what it actually means to -"run a board" with one--or with two computers, or with three.) - -About twenty-five boards vanished into police custody in May 1990. -As we have seen, there are an estimated 30,000 boards in America today. -If we assume that one board in a hundred is up to no good with codes -and cards (which rather flatters the honesty of the board-using community), -then that would leave 2,975 outlaw boards untouched by Sundevil. -Sundevil seized about one tenth of one percent of all computer -bulletin boards in America. Seen objectively, this is something less -than a comprehensive assault. In 1990, Sundevil's organizers-- -the team at the Phoenix Secret Service office, and the Arizona -Attorney General's office-- had a list of at least THREE HUNDRED -boards that they considered fully deserving of search and seizure warrants. -The twenty-five boards actually seized were merely among the most obvious -and egregious of this much larger list of candidates. All these boards -had been examined beforehand--either by informants, who had passed printouts -to the Secret Service, or by Secret Service agents themselves, who not only -come equipped with modems but know how to use them. - -There were a number of motives for Sundevil. First, it offered -a chance to get ahead of the curve on wire-fraud crimes. -Tracking back credit-card ripoffs to their perpetrators -can be appallingly difficult. If these miscreants -have any kind of electronic sophistication, they can snarl -their tracks through the phone network into a mind-boggling, -untraceable mess, while still managing to "reach out and rob someone." -Boards, however, full of brags and boasts, codes and cards, -offer evidence in the handy congealed form. - -Seizures themselves--the mere physical removal of machines-- -tends to take the pressure off. During Sundevil, a large number -of code kids, warez d00dz, and credit card thieves would be deprived -of those boards--their means of community and conspiracy--in one swift blow. -As for the sysops themselves (commonly among the boldest offenders) -they would be directly stripped of their computer equipment, -and rendered digitally mute and blind. - -And this aspect of Sundevil was carried out with great success. -Sundevil seems to have been a complete tactical surprise-- -unlike the fragmentary and continuing seizures of the war on the -Legion of Doom, Sundevil was precisely timed and utterly overwhelming. -At least forty "computers" were seized during May 7, 8 and 9, 1990, -in Cincinnati, Detroit, Los Angeles, Miami, Newark, Phoenix, Tucson, -Richmond, San Diego, San Jose, Pittsburgh and San Francisco. -Some cities saw multiple raids, such as the five separate raids -in the New York City environs. Plano, Texas (essentially a suburb of -the Dallas/Fort Worth metroplex, and a hub of the telecommunications industry) -saw four computer seizures. Chicago, ever in the forefront, saw its own -local Sundevil raid, briskly carried out by Secret Service agents -Timothy Foley and Barbara Golden. - -Many of these raids occurred, not in the cities proper, -but in associated white-middle class suburbs--places like -Mount Lebanon, Pennsylvania and Clark Lake, Michigan. -There were a few raids on offices; most took place in people's homes, -the classic hacker basements and bedrooms. - -The Sundevil raids were searches and seizures, not a group of mass arrests. -There were only four arrests during Sundevil. "Tony the Trashman," -a longtime teenage bete noire of the Arizona Racketeering unit, -was arrested in Tucson on May 9. "Dr. Ripco," sysop of an outlaw board -with the misfortune to exist in Chicago itself, was also arrested-- -on illegal weapons charges. Local units also arrested a 19-year-old -female phone phreak named "Electra" in Pennsylvania, and a male juvenile -in California. Federal agents however were not seeking arrests, but computers. - -Hackers are generally not indicted (if at all) until the evidence -in their seized computers is evaluated--a process that can take weeks, -months--even years. When hackers are arrested on the spot, it's generally -an arrest for other reasons. Drugs and/or illegal weapons show up in a good -third of anti-hacker computer seizures (though not during Sundevil). - -That scofflaw teenage hackers (or their parents) should have marijuana -in their homes is probably not a shocking revelation, but the surprisingly -common presence of illegal firearms in hacker dens is a bit disquieting. -A Personal Computer can be a great equalizer for the techno-cowboy-- -much like that more traditional American "Great Equalizer," -the Personal Sixgun. Maybe it's not all that surprising -that some guy obsessed with power through illicit technology -would also have a few illicit high-velocity-impact devices around. -An element of the digital underground particularly dotes on those -"anarchy philes," and this element tends to shade into the crackpot milieu -of survivalists, gun-nuts, anarcho-leftists and the ultra-libertarian -right-wing. - -This is not to say that hacker raids to date have uncovered any -major crack-dens or illegal arsenals; but Secret Service agents -do not regard "hackers" as "just kids." They regard hackers as -unpredictable people, bright and slippery. It doesn't help matters -that the hacker himself has been "hiding behind his keyboard" -all this time. Commonly, police have no idea what he looks like. -This makes him an unknown quantity, someone best treated with -proper caution. - -To date, no hacker has come out shooting, though they do sometimes brag on -boards that they will do just that. Threats of this sort are taken seriously. -Secret Service hacker raids tend to be swift, comprehensive, well-manned -(even over-manned); and agents generally burst through every door -in the home at once, sometimes with drawn guns. Any potential resistance -is swiftly quelled. Hacker raids are usually raids on people's homes. -It can be a very dangerous business to raid an American home; -people can panic when strangers invade their sanctum. Statistically speaking, -the most dangerous thing a policeman can do is to enter someone's home. -(The second most dangerous thing is to stop a car in traffic.) -People have guns in their homes. More cops are hurt in homes -than are ever hurt in biker bars or massage parlors. - -But in any case, no one was hurt during Sundevil, -or indeed during any part of the Hacker Crackdown. - -Nor were there any allegations of any physical mistreatment of a suspect. -Guns were pointed, interrogations were sharp and prolonged; but no one -in 1990 claimed any act of brutality by any crackdown raider. - -In addition to the forty or so computers, Sundevil reaped floppy disks -in particularly great abundance--an estimated 23,000 of them, which -naturally included every manner of illegitimate data: pirated games, -stolen codes, hot credit card numbers, the complete text and software -of entire pirate bulletin-boards. These floppy disks, which remain -in police custody today, offer a gigantic, almost embarrassingly -rich source of possible criminal indictments. These 23,000 floppy disks -also include a thus-far unknown quantity of legitimate computer games, -legitimate software, purportedly "private" mail from boards, -business records, and personal correspondence of all kinds. - -Standard computer-crime search warrants lay great emphasis on seizing -written documents as well as computers--specifically including photocopies, -computer printouts, telephone bills, address books, logs, notes, -memoranda and correspondence. In practice, this has meant that diaries, -gaming magazines, software documentation, nonfiction books on hacking -and computer security, sometimes even science fiction novels, have all -vanished out the door in police custody. A wide variety of electronic items -have been known to vanish as well, including telephones, televisions, answering -machines, Sony Walkmans, desktop printers, compact disks, and audiotapes. - -No fewer than 150 members of the Secret Service were sent into -the field during Sundevil. They were commonly accompanied by -squads of local and/or state police. Most of these officers-- -especially the locals--had never been on an anti-hacker raid before. -(This was one good reason, in fact, why so many of them were invited along -in the first place.) Also, the presence of a uniformed police officer -assures the raidees that the people entering their homes are, in fact, police. -Secret Service agents wear plain clothes. So do the telco security experts -who commonly accompany the Secret Service on raids (and who make no particular -effort to identify themselves as mere employees of telephone companies). - -A typical hacker raid goes something like this. First, police storm in -rapidly, through every entrance, with overwhelming force, -in the assumption that this tactic will keep casualties to a minimum. -Second, possible suspects are immediately removed from the vicinity -of any and all computer systems, so that they will have no chance -to purge or destroy computer evidence. Suspects are herded into a room -without computers, commonly the living room, and kept under guard-- -not ARMED guard, for the guns are swiftly holstered, but under guard -nevertheless. They are presented with the search warrant and warned -that anything they say may be held against them. Commonly they have -a great deal to say, especially if they are unsuspecting parents. - -Somewhere in the house is the "hot spot"--a computer tied to a phone -line (possibly several computers and several phones). Commonly it's -a teenager's bedroom, but it can be anywhere in the house; -there may be several such rooms. This "hot spot" is put in charge -of a two-agent team, the "finder" and the "recorder." The "finder" -is computer-trained, commonly the case agent who has actually obtained -the search warrant from a judge. He or she understands what is being sought, -and actually carries out the seizures: unplugs machines, opens drawers, -desks, files, floppy-disk containers, etc. The "recorder" photographs -all the equipment, just as it stands--especially the tangle of -wired connections in the back, which can otherwise be a real nightmare -to restore. The recorder will also commonly photograph every room -in the house, lest some wily criminal claim that the police had robbed him -during the search. Some recorders carry videocams or tape recorders; -however, it's more common for the recorder to simply take written notes. -Objects are described and numbered as the finder seizes them, generally -on standard preprinted police inventory forms. - -Even Secret Service agents were not, and are not, expert computer users. -They have not made, and do not make, judgements on the fly about potential -threats posed by various forms of equipment. They may exercise discretion; -they may leave Dad his computer, for instance, but they don't HAVE to. -Standard computer-crime search warrants, which date back to the early 80s, -use a sweeping language that targets computers, most anything attached -to a computer, most anything used to operate a computer--most anything -that remotely resembles a computer--plus most any and all written documents -surrounding it. Computer-crime investigators have strongly urged agents -to seize the works. - -In this sense, Operation Sundevil appears to have been a complete success. -Boards went down all over America, and were shipped en masse to the computer -investigation lab of the Secret Service, in Washington DC, along with the -23,000 floppy disks and unknown quantities of printed material. - -But the seizure of twenty-five boards, and the multi-megabyte mountains -of possibly useful evidence contained in these boards (and in their owners' -other computers, also out the door), were far from the only motives for -Operation Sundevil. An unprecedented action of great ambition and size, -Sundevil's motives can only be described as political. It was a -public-relations effort, meant to pass certain messages, meant to make -certain situations clear: both in the mind of the general public, -and in the minds of various constituencies of the electronic community. - - First --and this motivation was vital--a "message" would be sent from -law enforcement to the digital underground. This very message was recited -in so many words by Garry M. Jenkins, the Assistant Director of the -US Secret Service, at the Sundevil press conference in Phoenix on -May 9, 1990, immediately after the raids. In brief, hackers were -mistaken in their foolish belief that they could hide behind the -"relative anonymity of their computer terminals." On the contrary, -they should fully understand that state and federal cops were -actively patrolling the beat in cyberspace--that they were -on the watch everywhere, even in those sleazy and secretive -dens of cybernetic vice, the underground boards. - -This is not an unusual message for police to publicly convey to crooks. -The message is a standard message; only the context is new. - -In this respect, the Sundevil raids were the digital equivalent -of the standard vice-squad crackdown on massage parlors, porno bookstores, -head-shops, or floating crap-games. There may be few or no arrests in a raid -of this sort; no convictions, no trials, no interrogations. In cases of this -sort, police may well walk out the door with many pounds of sleazy magazines, -X-rated videotapes, sex toys, gambling equipment, baggies of marijuana. . . . - -Of course, if something truly horrendous is discovered by the raiders, -there will be arrests and prosecutions. Far more likely, however, -there will simply be a brief but sharp disruption of the closed -and secretive world of the nogoodniks. There will be "street hassle." -"Heat." "Deterrence." And, of course, the immediate loss of the seized goods. -It is very unlikely that any of this seized material will ever be returned. -Whether charged or not, whether convicted or not, the perpetrators will -almost surely lack the nerve ever to ask for this stuff to be given back. - -Arrests and trials--putting people in jail--may involve all kinds of -formal legalities; but dealing with the justice system is far from the only -task of police. Police do not simply arrest people. They don't simply -put people in jail. That is not how the police perceive their jobs. -Police "protect and serve." Police "keep the peace," they "keep public order." -Like other forms of public relations, keeping public order is not an -exact science. Keeping public order is something of an art-form. - -If a group of tough-looking teenage hoodlums was loitering on a street-corner, -no one would be surprised to see a street-cop arrive and sternly order -them to "break it up." On the contrary, the surprise would come if one -of these ne'er-do-wells stepped briskly into a phone-booth, -called a civil rights lawyer, and instituted a civil suit -in defense of his Constitutional rights of free speech -and free assembly. But something much along this line -was one of the many anomolous outcomes of the Hacker Crackdown. - -Sundevil also carried useful "messages" for other constituents of -the electronic community. These messages may not have been read -aloud from the Phoenix podium in front of the press corps, -but there was little mistaking their meaning. There was a message -of reassurance for the primary victims of coding and carding: -the telcos, and the credit companies. Sundevil was greeted with joy -by the security officers of the electronic business community. -After years of high-tech harassment and spiralling revenue losses, -their complaints of rampant outlawry were being taken seriously by -law enforcement. No more head-scratching or dismissive shrugs; -no more feeble excuses about "lack of computer-trained officers" or -the low priority of "victimless" white-collar telecommunication crimes. - -Computer-crime experts have long believed that computer-related offenses -are drastically under-reported. They regard this as a major open scandal -of their field. Some victims are reluctant to come forth, because they -believe that police and prosecutors are not computer-literate, -and can and will do nothing. Others are embarrassed by -their vulnerabilities, and will take strong measures -to avoid any publicity; this is especially true of banks, -who fear a loss of investor confidence should an embezzlement-case -or wire-fraud surface. And some victims are so helplessly confused -by their own high technology that they never even realize that -a crime has occurred--even when they have been fleeced to the bone. - -The results of this situation can be dire. -Criminals escape apprehension and punishment. -The computer-crime units that do exist, can't get work. -The true scope of computer-crime: its size, its real nature, -the scope of its threats, and the legal remedies for it-- -all remain obscured. - -Another problem is very little publicized, but it is a cause -of genuine concern. Where there is persistent crime, -but no effective police protection, then vigilantism can result. -Telcos, banks, credit companies, the major corporations who -maintain extensive computer networks vulnerable to hacking ---these organizations are powerful, wealthy, and -politically influential. They are disinclined to be -pushed around by crooks (or by most anyone else, -for that matter). They often maintain well-organized -private security forces, commonly run by -experienced veterans of military and police units, -who have left public service for the greener pastures -of the private sector. For police, the corporate -security manager can be a powerful ally; but if this -gentleman finds no allies in the police, and the -pressure is on from his board-of-directors, -he may quietly take certain matters into his own hands. - -Nor is there any lack of disposable hired-help in the -corporate security business. Private security agencies-- -the `security business' generally--grew explosively in the 1980s. -Today there are spooky gumshoed armies of "security consultants," -"rent-a- cops," "private eyes," "outside experts"--every manner -of shady operator who retails in "results" and discretion. -Or course, many of these gentlemen and ladies may be paragons -of professional and moral rectitude. But as anyone -who has read a hard-boiled detective novel knows, -police tend to be less than fond of this sort -of private-sector competition. - -Companies in search of computer-security have even been -known to hire hackers. Police shudder at this prospect. - -Police treasure good relations with the business community. -Rarely will you see a policeman so indiscreet as to allege -publicly that some major employer in his state or city has succumbed -to paranoia and gone off the rails. Nevertheless, -police --and computer police in particular--are aware -of this possibility. Computer-crime police can and do -spend up to half of their business hours just doing -public relations: seminars, "dog and pony shows," -sometimes with parents' groups or computer users, -but generally with their core audience: the likely -victims of hacking crimes. These, of course, are telcos, -credit card companies and large computer-equipped corporations. -The police strongly urge these people, as good citizens, -to report offenses and press criminal charges; -they pass the message that there is someone in authority who cares, -understands, and, best of all, will take useful action -should a computer-crime occur. - -But reassuring talk is cheap. Sundevil offered action. - -The final message of Sundevil was intended for internal consumption -by law enforcement. Sundevil was offered as proof that the community -of American computer-crime police had come of age. Sundevil was -proof that enormous things like Sundevil itself could now be accomplished. -Sundevil was proof that the Secret Service and its local law-enforcement -allies could act like a well-oiled machine--(despite the hampering use -of those scrambled phones). It was also proof that the Arizona Organized -Crime and Racketeering Unit--the sparkplug of Sundevil--ranked with the best -in the world in ambition, organization, and sheer conceptual daring. - -And, as a final fillip, Sundevil was a message from the Secret Service -to their longtime rivals in the Federal Bureau of Investigation. -By Congressional fiat, both USSS and FBI formally share jurisdiction -over federal computer-crimebusting activities. Neither of these groups -has ever been remotely happy with this muddled situation. It seems to -suggest that Congress cannot make up its mind as to which of these groups -is better qualified. And there is scarcely a G-man or a Special Agent -anywhere without a very firm opinion on that topic. - -# - -For the neophyte, one of the most puzzling aspects of the crackdown -on hackers is why the United States Secret Service has anything at all -to do with this matter. - -The Secret Service is best known for its primary public role: -its agents protect the President of the United States. -They also guard the President's family, the Vice President and his family, -former Presidents, and Presidential candidates. They sometimes guard -foreign dignitaries who are visiting the United States, especially foreign -heads of state, and have been known to accompany American officials -on diplomatic missions overseas. - -Special Agents of the Secret Service don't wear uniforms, but the -Secret Service also has two uniformed police agencies. There's the -former White House Police (now known as the Secret Service Uniformed Division, -since they currently guard foreign embassies in Washington, as well as the -White House itself). And there's the uniformed Treasury Police Force. - -The Secret Service has been charged by Congress with a number -of little-known duties. They guard the precious metals in Treasury vaults. -They guard the most valuable historical documents of the United States: -originals of the Constitution, the Declaration of Independence, -Lincoln's Second Inaugural Address, an American-owned copy of -the Magna Carta, and so forth. Once they were assigned to guard -the Mona Lisa, on her American tour in the 1960s. - -The entire Secret Service is a division of the Treasury Department. -Secret Service Special Agents (there are about 1,900 of them) -are bodyguards for the President et al, but they all work for the Treasury. -And the Treasury (through its divisions of the U.S. Mint and the -Bureau of Engraving and Printing) prints the nation's money. - -As Treasury police, the Secret Service guards the nation's currency; -it is the only federal law enforcement agency with direct jurisdiction -over counterfeiting and forgery. It analyzes documents for authenticity, -and its fight against fake cash is still quite lively (especially since -the skilled counterfeiters of Medellin, Columbia have gotten into the act). -Government checks, bonds, and other obligations, which exist in untold -millions and are worth untold billions, are common targets for forgery, -which the Secret Service also battles. It even handles forgery -of postage stamps. - -But cash is fading in importance today as money has become electronic. -As necessity beckoned, the Secret Service moved from fighting the -counterfeiting of paper currency and the forging of checks, -to the protection of funds transferred by wire. - -From wire-fraud, it was a simple skip-and-jump to what is formally -known as "access device fraud." Congress granted the Secret Service -the authority to investigate "access device fraud" under Title 18 -of the United States Code (U.S.C. Section 1029). - -The term "access device" seems intuitively simple. It's some kind -of high-tech gizmo you use to get money with. It makes good sense -to put this sort of thing in the charge of counterfeiting and -wire-fraud experts. - -However, in Section 1029, the term "access device" is very -generously defined. An access device is: "any card, plate, -code, account number, or other means of account access -that can be used, alone or in conjunction with another access device, -to obtain money, goods, services, or any other thing of value, -or that can be used to initiate a transfer of funds." - -"Access device" can therefore be construed to include credit cards -themselves (a popular forgery item nowadays). It also includes credit card -account NUMBERS, those standards of the digital underground. The same goes -for telephone charge cards (an increasingly popular item with telcos, -who are tired of being robbed of pocket change by phone-booth thieves). -And also telephone access CODES, those OTHER standards of the digital -underground. (Stolen telephone codes may not "obtain money," but they -certainly do obtain valuable "services," which is specifically forbidden -by Section 1029.) - -We can now see that Section 1029 already pits the United States Secret Service -directly against the digital underground, without any mention at all of -the word "computer." - -Standard phreaking devices, like "blue boxes," used to steal phone service -from old-fashioned mechanical switches, are unquestionably "counterfeit -access devices." Thanks to Sec.1029, it is not only illegal to USE -counterfeit access devices, but it is even illegal to BUILD them. -"Producing," "designing" "duplicating" or "assembling" blue boxes -are all federal crimes today, and if you do this, the Secret Service -has been charged by Congress to come after you. - -Automatic Teller Machines, which replicated all over America during the 1980s, -are definitely "access devices," too, and an attempt to tamper with their -punch-in codes and plastic bank cards falls directly under Sec. 1029. - -Section 1029 is remarkably elastic. Suppose you find a computer password -in somebody's trash. That password might be a "code"--it's certainly a -"means of account access." Now suppose you log on to a computer -and copy some software for yourself. You've certainly obtained -"service" (computer service) and a "thing of value" (the software). -Suppose you tell a dozen friends about your swiped password, -and let them use it, too. Now you're "trafficking in unauthorized -access devices." And when the Prophet, a member of the Legion of Doom, -passed a stolen telephone company document to Knight Lightning -at Phrack magazine, they were both charged under Sec. 1029! - -There are two limitations on Section 1029. First, the offense must -"affect interstate or foreign commerce" in order to become a matter -of federal jurisdiction. The term "affecting commerce" is not well defined; -but you may take it as a given that the Secret Service can take an interest -if you've done most anything that happens to cross a state line. -State and local police can be touchy about their jurisdictions, -and can sometimes be mulish when the feds show up. But when it comes -to computer-crime, the local police are pathetically grateful -for federal help--in fact they complain that they can't get enough of it. -If you're stealing long-distance service, you're almost certainly crossing -state lines, and you're definitely "affecting the interstate commerce" -of the telcos. And if you're abusing credit cards by ordering stuff -out of glossy catalogs from, say, Vermont, you're in for it. - -The second limitation is money. As a rule, the feds don't pursue -penny-ante offenders. Federal judges will dismiss cases that appear -to waste their time. Federal crimes must be serious; Section 1029 -specifies a minimum loss of a thousand dollars. - -We now come to the very next section of Title 18, which is Section 1030, -"Fraud and related activity in connection with computers." This statute -gives the Secret Service direct jurisdiction over acts of computer intrusion. -On the face of it, the Secret Service would now seem to command the field. -Section 1030, however, is nowhere near so ductile as Section 1029. - -The first annoyance is Section 1030(d), which reads: - -"(d) The United States Secret Service shall, -IN ADDITION TO ANY OTHER AGENCY HAVING SUCH AUTHORITY, -have the authority to investigate offenses under this section. -Such authority of the United States Secret Service shall be -exercised in accordance with an agreement which shall be entered -into by the Secretary of the Treasury AND THE ATTORNEY GENERAL." -(Author's italics.) [Represented by capitals.] - -The Secretary of the Treasury is the titular head of the Secret Service, -while the Attorney General is in charge of the FBI. In Section (d), -Congress shrugged off responsibility for the computer-crime turf-battle -between the Service and the Bureau, and made them fight it out all -by themselves. The result was a rather dire one for the Secret Service, -for the FBI ended up with exclusive jurisdiction over computer break-ins -having to do with national security, foreign espionage, federally insured -banks, and U.S. military bases, while retaining joint jurisdiction over -all the other computer intrusions. Essentially, when it comes to Section 1030, -the FBI not only gets the real glamor stuff for itself, but can peer over the -shoulder of the Secret Service and barge in to meddle whenever it suits them. - -The second problem has to do with the dicey term -"Federal interest computer." Section 1030(a)(2) -makes it illegal to "access a computer without authorization" -if that computer belongs to a financial institution or an issuer -of credit cards (fraud cases, in other words). Congress was quite -willing to give the Secret Service jurisdiction over -money-transferring computers, but Congress balked at -letting them investigate any and all computer intrusions. -Instead, the USSS had to settle for the money machines -and the "Federal interest computers." A "Federal interest computer" -is a computer which the government itself owns, or is using. -Large networks of interstate computers, linked over state lines, -are also considered to be of "Federal interest." (This notion of -"Federal interest" is legally rather foggy and has never been -clearly defined in the courts. The Secret Service has never yet -had its hand slapped for investigating computer break-ins that were NOT -of "Federal interest," but conceivably someday this might happen.) - -So the Secret Service's authority over "unauthorized access" -to computers covers a lot of territory, but by no means the -whole ball of cyberspatial wax. If you are, for instance, -a LOCAL computer retailer, or the owner of a LOCAL bulletin -board system, then a malicious LOCAL intruder can break in, -crash your system, trash your files and scatter viruses, -and the U.S. Secret Service cannot do a single thing about it. - -At least, it can't do anything DIRECTLY. But the Secret Service -will do plenty to help the local people who can. - -The FBI may have dealt itself an ace off the bottom of the deck -when it comes to Section 1030; but that's not the whole story; -that's not the street. What's Congress thinks is one thing, -and Congress has been known to change its mind. The REAL -turf-struggle is out there in the streets where it's happening. -If you're a local street-cop with a computer problem, -the Secret Service wants you to know where you can find -the real expertise. While the Bureau crowd are off having -their favorite shoes polished--(wing-tips)--and making derisive -fun of the Service's favorite shoes--("pansy-ass tassels")-- -the tassel-toting Secret Service has a crew of ready-and-able -hacker-trackers installed in the capital of every state in the Union. -Need advice? They'll give you advice, or at least point you in -the right direction. Need training? They can see to that, too. - -If you're a local cop and you call in the FBI, the FBI -(as is widely and slanderously rumored) will order you around -like a coolie, take all the credit for your busts, -and mop up every possible scrap of reflected glory. -The Secret Service, on the other hand, doesn't brag a lot. -They're the quiet types. VERY quiet. Very cool. Efficient. -High-tech. Mirrorshades, icy stares, radio ear-plugs, -an Uzi machine-pistol tucked somewhere in that well-cut jacket. -American samurai, sworn to give their lives to protect our President. -"The granite agents." Trained in martial arts, absolutely fearless. -Every single one of 'em has a top-secret security clearance. -Something goes a little wrong, you're not gonna hear any whining -and moaning and political buck-passing out of these guys. - -The facade of the granite agent is not, of course, the reality. -Secret Service agents are human beings. And the real glory -in Service work is not in battling computer crime--not yet, -anyway--but in protecting the President. The real glamour -of Secret Service work is in the White House Detail. -If you're at the President's side, then the kids and the wife -see you on television; you rub shoulders with the most powerful -people in the world. That's the real heart of Service work, -the number one priority. More than one computer investigation -has stopped dead in the water when Service agents vanished at -the President's need. - -There's romance in the work of the Service. The intimate access -to circles of great power; the esprit-de-corps of a highly trained -and disciplined elite; the high responsibility of defending the -Chief Executive; the fulfillment of a patriotic duty. And as police -work goes, the pay's not bad. But there's squalor in Service work, too. -You may get spat upon by protesters howling abuse--and if they get violent, -if they get too close, sometimes you have to knock one of them down-- -discreetly. - -The real squalor in Service work is drudgery such as "the quarterlies," -traipsing out four times a year, year in, year out, to interview the various -pathetic wretches, many of them in prisons and asylums, who have seen fit -to threaten the President's life. And then there's the grinding stress -of searching all those faces in the endless bustling crowds, looking for -hatred, looking for psychosis, looking for the tight, nervous face -of an Arthur Bremer, a Squeaky Fromme, a Lee Harvey Oswald. -It's watching all those grasping, waving hands for sudden movements, -while your ears strain at your radio headphone for the long-rehearsed -cry of "Gun!" - -It's poring, in grinding detail, over the biographies of every rotten -loser who ever shot at a President. It's the unsung work of the -Protective Research Section, who study scrawled, anonymous death threats -with all the meticulous tools of anti-forgery techniques. - -And it's maintaining the hefty computerized files on anyone -who ever threatened the President's life. Civil libertarians -have become increasingly concerned at the Government's use -of computer files to track American citizens--but the -Secret Service file of potential Presidential assassins, -which has upward of twenty thousand names, rarely causes -a peep of protest. If you EVER state that you intend to -kill the President, the Secret Service will want to know -and record who you are, where you are, what you are, -and what you're up to. If you're a serious threat-- -if you're officially considered "of protective interest"-- -then the Secret Service may well keep tabs on you -for the rest of your natural life. - -Protecting the President has first call on all the Service's resources. -But there's a lot more to the Service's traditions and history than -standing guard outside the Oval Office. - -The Secret Service is the nation's oldest general federal -law-enforcement agency. Compared to the Secret Service, -the FBI are new-hires and the CIA are temps. The Secret Service -was founded 'way back in 1865, at the suggestion of Hugh McCulloch, -Abraham Lincoln's Secretary of the Treasury. McCulloch wanted -a specialized Treasury police to combat counterfeiting. -Abraham Lincoln agreed that this seemed a good idea, and, -with a terrible irony, Abraham Lincoln was shot that -very night by John Wilkes Booth. - -The Secret Service originally had nothing to do with protecting Presidents. -They didn't take this on as a regular assignment until after the Garfield -assassination in 1881. And they didn't get any Congressional money for it -until President McKinley was shot in 1901. The Service was originally -designed for one purpose: destroying counterfeiters. - -# - -There are interesting parallels between the Service's -nineteenth-century entry into counterfeiting, -and America's twentieth-century entry into computer-crime. - -In 1865, America's paper currency was a terrible muddle. -Security was drastically bad. Currency was printed on the spot -by local banks in literally hundreds of different designs. -No one really knew what the heck a dollar bill was supposed to look like. -Bogus bills passed easily. If some joker told you that a one-dollar bill -from the Railroad Bank of Lowell, Massachusetts had a woman leaning on -a shield, with a locomotive, a cornucopia, a compass, various agricultural -implements, a railroad bridge, and some factories, then you pretty much had -to take his word for it. (And in fact he was telling the truth!) - -SIXTEEN HUNDRED local American banks designed and printed their own -paper currency, and there were no general standards for security. -Like a badly guarded node in a computer network, badly designed bills -were easy to fake, and posed a security hazard for the entire monetary system. - -No one knew the exact extent of the threat to the currency. -There were panicked estimates that as much as a third of -the entire national currency was faked. Counterfeiters-- -known as "boodlers" in the underground slang of the time-- -were mostly technically skilled printers who had gone to the bad. -Many had once worked printing legitimate currency. -Boodlers operated in rings and gangs. Technical experts -engraved the bogus plates--commonly in basements in New York City. -Smooth confidence men passed large wads of high-quality, -high-denomination fakes, including the really sophisticated stuff-- -government bonds, stock certificates, and railway shares. -Cheaper, botched fakes were sold or sharewared to low-level -gangs of boodler wannabes. (The really cheesy lowlife boodlers -merely upgraded real bills by altering face values, -changing ones to fives, tens to hundreds, and so on.) - -The techniques of boodling were little-known and regarded -with a certain awe by the mid- nineteenth-century public. -The ability to manipulate the system for rip-off seemed -diabolically clever. As the skill and daring of the -boodlers increased, the situation became intolerable. -The federal government stepped in, and began offering -its own federal currency, which was printed in fancy green ink, -but only on the back--the original "greenbacks." And at first, -the improved security of the well-designed, well-printed -federal greenbacks seemed to solve the problem; but then -the counterfeiters caught on. Within a few years things were -worse than ever: a CENTRALIZED system where ALL security was bad! - -The local police were helpless. The Government tried offering -blood money to potential informants, but this met with little success. -Banks, plagued by boodling, gave up hope of police help and hired -private security men instead. Merchants and bankers queued up -by the thousands to buy privately-printed manuals on currency security, -slim little books like Laban Heath's INFALLIBLE GOVERNMENT -COUNTERFEIT DETECTOR. The back of the book offered Laban Heath's -patent microscope for five bucks. - -Then the Secret Service entered the picture. The first agents -were a rough and ready crew. Their chief was one William P. Wood, -a former guerilla in the Mexican War who'd won a reputation busting -contractor fraudsters for the War Department during the Civil War. -Wood, who was also Keeper of the Capital Prison, had a sideline -as a counterfeiting expert, bagging boodlers for the federal bounty money. - -Wood was named Chief of the new Secret Service in July 1865. -There were only ten Secret Service agents in all: Wood himself, -a handful who'd worked for him in the War Department, and a few -former private investigators--counterfeiting experts--whom Wood -had won over to public service. (The Secret Service of 1865 was -much the size of the Chicago Computer Fraud Task Force or the -Arizona Racketeering Unit of 1990.) These ten "Operatives" -had an additional twenty or so "Assistant Operatives" and "Informants." -Besides salary and per diem, each Secret Service employee received -a whopping twenty-five dollars for each boodler he captured. - -Wood himself publicly estimated that at least HALF of America's currency -was counterfeit, a perhaps pardonable perception. Within a year the -Secret Service had arrested over 200 counterfeiters. They busted about -two hundred boodlers a year for four years straight. - -Wood attributed his success to travelling fast and light, hitting the -bad-guys hard, and avoiding bureaucratic baggage. "Because my raids -were made without military escort and I did not ask the assistance -of state officers, I surprised the professional counterfeiter." - -Wood's social message to the once-impudent boodlers bore an eerie ring -of Sundevil: "It was also my purpose to convince such characters that -it would no longer be healthy for them to ply their vocation without -being handled roughly, a fact they soon discovered." - -William P. Wood, the Secret Service's guerilla pioneer, -did not end well. He succumbed to the lure of aiming for -the really big score. The notorious Brockway Gang of New York City, -headed by William E. Brockway, the "King of the Counterfeiters," -had forged a number of government bonds. They'd passed these -brilliant fakes on the prestigious Wall Street investment -firm of Jay Cooke and Company. The Cooke firm were frantic -and offered a huge reward for the forgers' plates. - -Laboring diligently, Wood confiscated the plates -(though not Mr. Brockway) and claimed the reward. -But the Cooke company treacherously reneged. -Wood got involved in a down-and-dirty lawsuit -with the Cooke capitalists. Wood's boss, -Secretary of the Treasury McCulloch, felt that -Wood's demands for money and glory were unseemly, -and even when the reward money finally came through, -McCulloch refused to pay Wood anything. -Wood found himself mired in a seemingly endless -round of federal suits and Congressional lobbying. - -Wood never got his money. And he lost his job to boot. -He resigned in 1869. - -Wood's agents suffered, too. On May 12, 1869, the second Chief -of the Secret Service took over, and almost immediately fired -most of Wood's pioneer Secret Service agents: Operatives, -Assistants and Informants alike. The practice of receiving $25 -per crook was abolished. And the Secret Service began the long, -uncertain process of thorough professionalization. - -Wood ended badly. He must have felt stabbed in the back. -In fact his entire organization was mangled. - -On the other hand, William P. Wood WAS the first head of the Secret Service. -William Wood was the pioneer. People still honor his name. Who remembers -the name of the SECOND head of the Secret Service? - -As for William Brockway (also known as "Colonel Spencer"), -he was finally arrested by the Secret Service in 1880. -He did five years in prison, got out, and was still boodling -at the age of seventy-four. - -# - -Anyone with an interest in Operation Sundevil-- -or in American computer-crime generally-- -could scarcely miss the presence of Gail Thackeray, -Assistant Attorney General of the State of Arizona. -Computer-crime training manuals often cited -Thackeray's group and her work; she was the -highest-ranking state official to specialize -in computer-related offenses. Her name had been -on the Sundevil press release (though modestly ranked -well after the local federal prosecuting attorney and -the head of the Phoenix Secret Service office). - -As public commentary, and controversy, began to mount -about the Hacker Crackdown, this Arizonan state official -began to take a higher and higher public profile. -Though uttering almost nothing specific about -the Sundevil operation itself, she coined some -of the most striking soundbites of the growing propaganda war: -"Agents are operating in good faith, and I don't think -you can say that for the hacker community," was one. -Another was the memorable "I am not a mad dog prosecutor" -(Houston Chronicle, Sept 2, 1990.) In the meantime, -the Secret Service maintained its usual extreme discretion; -the Chicago Unit, smarting from the backlash -of the Steve Jackson scandal, had gone completely to earth. - -As I collated my growing pile of newspaper clippings, -Gail Thackeray ranked as a comparative fount of public -knowledge on police operations. - -I decided that I had to get to know Gail Thackeray. -I wrote to her at the Arizona Attorney General's Office. -Not only did she kindly reply to me, but, to my astonishment, -she knew very well what "cyberpunk" science fiction was. - -Shortly after this, Gail Thackeray lost her job. -And I temporarily misplaced my own career as -a science-fiction writer, to become a full-time -computer-crime journalist. In early March, 1991, -I flew to Phoenix, Arizona, to interview Gail Thackeray -for my book on the hacker crackdown. - -# - -"Credit cards didn't used to cost anything to get," -says Gail Thackeray. "Now they cost forty bucks-- -and that's all just to cover the costs from RIP-OFF ARTISTS." - -Electronic nuisance criminals are parasites. -One by one they're not much harm, no big deal. -But they never come just one by one. They come in swarms, -heaps, legions, sometimes whole subcultures. And they bite. -Every time we buy a credit card today, we lose a little financial -vitality to a particular species of bloodsucker. - -What, in her expert opinion, are the worst forms of electronic crime, -I ask, consulting my notes. Is it--credit card fraud? Breaking into -ATM bank machines? Phone-phreaking? Computer intrusions? -Software viruses? Access-code theft? Records tampering? -Software piracy? Pornographic bulletin boards? -Satellite TV piracy? Theft of cable service? -It's a long list. By the time I reach the end -of it I feel rather depressed. - -"Oh no," says Gail Thackeray, leaning forward over the table, -her whole body gone stiff with energetic indignation, -"the biggest damage is telephone fraud. Fake sweepstakes, -fake charities. Boiler-room con operations. You could pay off -the national debt with what these guys steal. . . . -They target old people, they get hold of credit ratings -and demographics, they rip off the old and the weak." -The words come tumbling out of her. - -It's low-tech stuff, your everyday boiler-room fraud. -Grifters, conning people out of money over the phone, -have been around for decades. This is where the word "phony" came from! - -It's just that it's so much EASIER now, horribly facilitated by advances -in technology and the byzantine structure of the modern phone system. -The same professional fraudsters do it over and over, Thackeray tells me, -they hide behind dense onion-shells of fake companies. . . fake holding -corporations nine or ten layers deep, registered all over the map. -They get a phone installed under a false name in an empty safe-house. -And then they call-forward everything out of that phone to yet -another phone, a phone that may even be in another STATE. -And they don't even pay the charges on their phones; -after a month or so, they just split; set up somewhere else -in another Podunkville with the same seedy crew of veteran phone-crooks. -They buy or steal commercial credit card reports, slap them on the PC, -have a program pick out people over sixty-five who pay a lot to charities. -A whole subculture living off this, merciless folks on the con. - -"The `light-bulbs for the blind' people," Thackeray muses, -with a special loathing. "There's just no end to them." - -We're sitting in a downtown diner in Phoenix, Arizona. -It's a tough town, Phoenix. A state capital seeing some hard times. -Even to a Texan like myself, Arizona state politics seem rather baroque. -There was, and remains, endless trouble over the Martin Luther King holiday, -the sort of stiff-necked, foot-shooting incident for which Arizona politics -seem famous. There was Evan Mecham, the eccentric Republican millionaire -governor who was impeached, after reducing state government to a -ludicrous shambles. Then there was the national Keating scandal, -involving Arizona savings and loans, in which both of Arizona's -U.S. senators, DeConcini and McCain, played sadly prominent roles. - -And the very latest is the bizarre AzScam case, -in which state legislators were videotaped, -eagerly taking cash from an informant of the Phoenix city -police department, who was posing as a Vegas mobster. - -"Oh," says Thackeray cheerfully. "These people are amateurs here, -they thought they were finally getting to play with the big boys. -They don't have the least idea how to take a bribe! -It's not institutional corruption. It's not like back in Philly." - -Gail Thackeray was a former prosecutor in Philadelphia. -Now she's a former assistant attorney general of the State of Arizona. -Since moving to Arizona in 1986, she had worked under the aegis -of Steve Twist, her boss in the Attorney General's office. -Steve Twist wrote Arizona's pioneering computer crime laws -and naturally took an interest in seeing them enforced. -It was a snug niche, and Thackeray's Organized Crime and -Racketeering Unit won a national reputation for ambition -and technical knowledgeability. . . . Until the latest -election in Arizona. Thackeray's boss ran for the top -job, and lost. The victor, the new Attorney General, -apparently went to some pains to eliminate the bureaucratic -traces of his rival, including his pet group--Thackeray's group. -Twelve people got their walking papers. - -Now Thackeray's painstakingly assembled computer lab -sits gathering dust somewhere in the glass-and-concrete -Attorney General's HQ on 1275 Washington Street. -Her computer-crime books, her painstakingly garnered -back issues of phreak and hacker zines, all bought -at her own expense--are piled in boxes somewhere. -The State of Arizona is simply not particularly -interested in electronic racketeering at the moment. - -At the moment of our interview, Gail Thackeray, -officially unemployed, is working out of the county -sheriff's office, living on her savings, and prosecuting -several cases--working 60-hour weeks, just as always-- -for no pay at all. "I'm trying to train people," -she mutters. - -Half her life seems to be spent training people--merely pointing out, -to the naive and incredulous (such as myself) that this stuff -is ACTUALLY GOING ON OUT THERE. It's a small world, computer crime. -A young world. Gail Thackeray, a trim blonde Baby-Boomer who favors -Grand Canyon white-water rafting to kill some slow time, -is one of the world's most senior, most veteran "hacker-trackers." -Her mentor was Donn Parker, the California think-tank theorist -who got it all started `way back in the mid-70s, the "grandfather -of the field," "the great bald eagle of computer crime." - -And what she has learned, Gail Thackeray teaches. Endlessly. -Tirelessly. To anybody. To Secret Service agents and state police, -at the Glynco, Georgia federal training center. To local police, -on "roadshows" with her slide projector and notebook. -To corporate security personnel. To journalists. To parents. - -Even CROOKS look to Gail Thackeray for advice. -Phone-phreaks call her at the office. They know very -well who she is. They pump her for information -on what the cops are up to, how much they know. -Sometimes whole CROWDS of phone phreaks, -hanging out on illegal conference calls, will call Gail -Thackeray up. They taunt her. And, as always, -they boast. Phone-phreaks, real stone phone-phreaks, -simply CANNOT SHUT UP. They natter on for hours. - -Left to themselves, they mostly talk about the intricacies -of ripping-off phones; it's about as interesting as listening -to hot-rodders talk about suspension and distributor-caps. -They also gossip cruelly about each other. And when talking -to Gail Thackeray, they incriminate themselves. "I have tapes," -Thackeray says coolly. - -Phone phreaks just talk like crazy. "Dial-Tone" out in Alabama -has been known to spend half-an-hour simply reading stolen -phone-codes aloud into voice-mail answering machines. -Hundreds, thousands of numbers, recited in a monotone, -without a break--an eerie phenomenon. When arrested, -it's a rare phone phreak who doesn't inform at endless length -on everybody he knows. - -Hackers are no better. What other group of criminals, -she asks rhetorically, publishes newsletters and holds conventions? -She seems deeply nettled by the sheer brazenness of this behavior, -though to an outsider, this activity might make one wonder -whether hackers should be considered "criminals" at all. -Skateboarders have magazines, and they trespass a lot. -Hot rod people have magazines and they break speed limits -and sometimes kill people. . . . - -I ask her whether it would be any loss to society if phone phreaking -and computer hacking, as hobbies, simply dried up and blew away, -so that nobody ever did it again. - -She seems surprised. "No," she says swiftly. "Maybe a little. . . -in the old days. . .the MIT stuff. . . . But there's a lot of wonderful, -legal stuff you can do with computers now, you don't have to break into -somebody else's just to learn. You don't have that excuse. -You can learn all you like." - -Did you ever hack into a system? I ask. - -The trainees do it at Glynco. Just to demonstrate system vulnerabilities. -She's cool to the notion. Genuinely indifferent. - -"What kind of computer do you have?" - -"A Compaq 286LE," she mutters. - -"What kind do you WISH you had?" - -At this question, the unmistakable light of true hackerdom flares in -Gail Thackeray's eyes. She becomes tense, animated, the words pour out: -"An Amiga 2000 with an IBM card and Mac emulation! The most common hacker -machines are Amigas and Commodores. And Apples." If she had the Amiga, -she enthuses, she could run a whole galaxy of seized computer-evidence disks -on one convenient multifunctional machine. A cheap one, too. Not like the -old Attorney General lab, where they had an ancient CP/M machine, -assorted Amiga flavors and Apple flavors, a couple IBMS, all the -utility software. . .but no Commodores. The workstations down -at the Attorney General's are Wang dedicated word-processors. -Lame machines tied in to an office net--though at least they get -on- line to the Lexis and Westlaw legal data services. - -I don't say anything. I recognize the syndrome, though. -This computer-fever has been running through segments of -our society for years now. It's a strange kind of lust: -K-hunger, Meg-hunger; but it's a shared disease; -it can kill parties dead, as conversation spirals into -the deepest and most deviant recesses of software releases -and expensive peripherals. . . . The mark of the hacker beast. -I have it too. The whole "electronic community," whatever the hell -that is, has it. Gail Thackeray has it. Gail Thackeray is a hacker cop. -My immediate reaction is a strong rush of indignant pity: -WHY DOESN'T SOMEBODY BUY THIS WOMAN HER AMIGA?! -It's not like she's asking for a Cray X-MP -supercomputer mainframe; an Amiga's a sweet little -cookie-box thing. We're losing zillions in organized fraud; -prosecuting and defending a single hacker case in court can cost -a hundred grand easy. How come nobody can come up with four lousy grand -so this woman can do her job? For a hundred grand we could buy every -computer cop in America an Amiga. There aren't that many of 'em. - -Computers. The lust, the hunger, for computers. -The loyalty they inspire, the intense sense of possessiveness. -The culture they have bred. I myself am sitting in downtown Phoenix, -Arizona because it suddenly occurred to me that the police might-- -just MIGHT--come and take away my computer. The prospect of this, -the mere IMPLIED THREAT, was unbearable. It literally changed my life. -It was changing the lives of many others. Eventually it would change -everybody's life. - -Gail Thackeray was one of the top computer-crime people in America. -And I was just some novelist, and yet I had a better computer than hers. -PRACTICALLY EVERYBODY I KNEW had a better computer than Gail Thackeray -and her feeble laptop 286. It was like sending the sheriff in to clean -up Dodge City and arming her with a slingshot cut from an old rubber tire. - -But then again, you don't need a howitzer to enforce the law. -You can do a lot just with a badge. With a badge alone, -you can basically wreak havoc, take a terrible vengeance on wrongdoers. -Ninety percent of "computer crime investigation" is just "crime investigation:" -names, places, dossiers, modus operandi, search warrants, victims, -complainants, informants. . . . - -What will computer crime look like in ten years? Will it get better? -Did "Sundevil" send 'em reeling back in confusion? - -It'll be like it is now, only worse, she tells me with perfect conviction. -Still there in the background, ticking along, changing with the times: -the criminal underworld. It'll be like drugs are. Like our problems -with alcohol. All the cops and laws in the world never solved our problems -with alcohol. If there's something people want, a certain percentage -of them are just going to take it. Fifteen percent of the populace -will never steal. Fifteen percent will steal most anything not nailed down. -The battle is for the hearts and minds of the remaining seventy percent. - -And criminals catch on fast. If there's not "too steep a learning curve"-- -if it doesn't require a baffling amount of expertise and practice-- -then criminals are often some of the first through the gate of a -new technology. Especially if it helps them to hide. -They have tons of cash, criminals. The new communications tech-- -like pagers, cellular phones, faxes, Federal Express--were pioneered -by rich corporate people, and by criminals. In the early years -of pagers and beepers, dope dealers were so enthralled this technology -that owing a beeper was practically prima facie evidence of cocaine dealing. -CB radio exploded when the speed limit hit 55 and breaking the highway law -became a national pastime. Dope dealers send cash by Federal Express, -despite, or perhaps BECAUSE OF, the warnings in FedEx offices that tell you -never to try this. Fed Ex uses X-rays and dogs on their mail, -to stop drug shipments. That doesn't work very well. - -Drug dealers went wild over cellular phones. -There are simple methods of faking ID on cellular phones, -making the location of the call mobile, free of charge, -and effectively untraceable. Now victimized cellular -companies routinely bring in vast toll-lists of calls -to Colombia and Pakistan. - -Judge Greene's fragmentation of the phone company -is driving law enforcement nuts. Four thousand -telecommunications companies. Fraud skyrocketing. -Every temptation in the world available with a phone -and a credit card number. Criminals untraceable. -A galaxy of "new neat rotten things to do." - -If there were one thing Thackeray would like to have, -it would be an effective legal end-run through this new -fragmentation minefield. - -It would be a new form of electronic search warrant, -an "electronic letter of marque" to be issued by a judge. -It would create a new category of "electronic emergency." -Like a wiretap, its use would be rare, but it would cut -across state lines and force swift cooperation from all concerned. -Cellular, phone, laser, computer network, PBXes, AT&T, Baby Bells, -long-distance entrepreneurs, packet radio. Some document, -some mighty court-order, that could slice through four thousand -separate forms of corporate red-tape, and get her at once to -the source of calls, the source of email threats and viruses, -the sources of bomb threats, kidnapping threats. "From now on," -she says, "the Lindbergh baby will always die." - -Something that would make the Net sit still, if only for a moment. -Something that would get her up to speed. Seven league boots. -That's what she really needs. "Those guys move in nanoseconds -and I'm on the Pony Express." - -And then, too, there's the coming international angle. -Electronic crime has never been easy to localize, -to tie to a physical jurisdiction. And phone-phreaks -and hackers loathe boundaries, they jump them whenever they can. -The English. The Dutch. And the Germans, especially the ubiquitous -Chaos Computer Club. The Australians. They've all learned phone-phreaking -from America. It's a growth mischief industry. The multinational -networks are global, but governments and the police simply aren't. -Neither are the laws. Or the legal frameworks for citizen protection. - -One language is global, though--English. Phone phreaks speak English; -it's their native tongue even if they're Germans. English may have started -in England but now it's the Net language; it might as well be called "CNNese." - -Asians just aren't much into phone phreaking. They're the world masters -at organized software piracy. The French aren't into phone-phreaking either. -The French are into computerized industrial espionage. - -In the old days of the MIT righteous hackerdom, crashing systems -didn't hurt anybody. Not all that much, anyway. Not permanently. -Now the players are more venal. Now the consequences are worse. -Hacking will begin killing people soon. Already there are methods -of stacking calls onto 911 systems, annoying the police, and possibly -causing the death of some poor soul calling in with a genuine emergency. -Hackers in Amtrak computers, or air-traffic control computers, will kill -somebody someday. Maybe a lot of people. Gail Thackeray expects it. - -And the viruses are getting nastier. The "Scud" virus is the latest one out. -It wipes hard-disks. - -According to Thackeray, the idea that phone-phreaks are Robin Hoods is a fraud. -They don't deserve this repute. Basically, they pick on the weak. AT&T now -protects itself with the fearsome ANI (Automatic Number Identification) -trace capability. When AT&T wised up and tightened security generally, -the phreaks drifted into the Baby Bells. The Baby Bells lashed out in 1989 -and 1990, so the phreaks switched to smaller long-distance entrepreneurs. -Today, they are moving into locally owned PBXes and voice-mail systems, -which are full of security holes, dreadfully easy to hack. These victims -aren't the moneybags Sheriff of Nottingham or Bad King John, but small groups -of innocent people who find it hard to protect themselves, and who really -suffer from these depredations. Phone phreaks pick on the weak. They do it -for power. If it were legal, they wouldn't do it. They don't want service, -or knowledge, they want the thrill of power-tripping. There's plenty of -knowledge or service around if you're willing to pay. Phone phreaks don't pay, -they steal. It's because it is illegal that it feels like power, -that it gratifies their vanity. - -I leave Gail Thackeray with a handshake at the door of her office building-- -a vast International-Style office building downtown. The Sheriff's office -is renting part of it. I get the vague impression that quite a lot of the -building is empty--real estate crash. - -In a Phoenix sports apparel store, in a downtown mall, I meet -the "Sun Devil" himself. He is the cartoon mascot of -Arizona State University, whose football stadium, "Sundevil," -is near the local Secret Service HQ--hence the name Operation Sundevil. -The Sun Devil himself is named "Sparky." Sparky the Sun Devil is maroon -and bright yellow, the school colors. Sparky brandishes a three-tined -yellow pitchfork. He has a small mustache, pointed ears, a barbed tail, -and is dashing forward jabbing the air with the pitchfork, -with an expression of devilish glee. - -Phoenix was the home of Operation Sundevil. The Legion of Doom -ran a hacker bulletin board called "The Phoenix Project." -An Australian hacker named "Phoenix" once burrowed through -the Internet to attack Cliff Stoll, then bragged and boasted -about it to The New York Times. This net of coincidence -is both odd and meaningless. - -The headquarters of the Arizona Attorney General, Gail Thackeray's -former workplace, is on 1275 Washington Avenue. Many of the downtown -streets in Phoenix are named after prominent American presidents: -Washington, Jefferson, Madison. . . . - -After dark, all the employees go home to their suburbs. -Washington, Jefferson and Madison--what would be the -Phoenix inner city, if there were an inner city in this -sprawling automobile-bred town--become the haunts -of transients and derelicts. The homeless. The sidewalks -along Washington are lined with orange trees. -Ripe fallen fruit lies scattered like croquet balls -on the sidewalks and gutters. No one seems to be eating them. -I try a fresh one. It tastes unbearably bitter. - -The Attorney General's office, built in 1981 during the -Babbitt administration, is a long low two-story building -of white cement and wall-sized sheets of curtain-glass. -Behind each glass wall is a lawyer's office, quite open -and visible to anyone strolling by. Across the street -is a dour government building labelled simply ECONOMIC SECURITY, -something that has not been in great supply in the American -Southwest lately. - -The offices are about twelve feet square. They feature -tall wooden cases full of red-spined lawbooks; -Wang computer monitors; telephones; Post-it notes galore. -Also framed law diplomas and a general excess of bad -Western landscape art. Ansel Adams photos are a big favorite, -perhaps to compensate for the dismal specter of the parking lot, -two acres of striped black asphalt, which features gravel landscaping -and some sickly-looking barrel cacti. - -It has grown dark. Gail Thackeray has told me that the people -who work late here, are afraid of muggings in the parking lot. -It seems cruelly ironic that a woman tracing electronic racketeers -across the interstate labyrinth of Cyberspace should fear an assault -by a homeless derelict in the parking lot of her own workplace. - -Perhaps this is less than coincidence. Perhaps these two seemingly -disparate worlds are somehow generating one another. The poor and -disenfranchised take to the streets, while the rich and computer-equipped, -safe in their bedrooms, chatter over their modems. Quite often the derelicts -kick the glass out and break in to the lawyers' offices, if they see something -they need or want badly enough. - -I cross the parking lot to the street behind the Attorney General's office. -A pair of young tramps are bedding down on flattened sheets of cardboard, -under an alcove stretching over the sidewalk. One tramp wears a -glitter-covered T-shirt reading "CALIFORNIA" in Coca-Cola cursive. -His nose and cheeks look chafed and swollen; they glisten with -what seems to be Vaseline. The other tramp has a ragged long-sleeved -shirt and lank brown hair parted in the middle. They both wear blue jeans -coated in grime. They are both drunk. - -"You guys crash here a lot?" I ask them. - -They look at me warily. I am wearing black jeans, a black pinstriped -suit jacket and a black silk tie. I have odd shoes and a funny haircut. - -"It's our first time here," says the red-nosed tramp unconvincingly. -There is a lot of cardboard stacked here. More than any two people could use. - -"We usually stay at the Vinnie's down the street," says the brown-haired tramp, -puffing a Marlboro with a meditative air, as he sprawls with his head on -a blue nylon backpack. "The Saint Vincent's." - -"You know who works in that building over there?" I ask, pointing. - -The brown-haired tramp shrugs. "Some kind of attorneys, it says." - -We urge one another to take it easy. I give them five bucks. - -A block down the street I meet a vigorous workman who is wheeling along -some kind of industrial trolley; it has what appears to be a tank of -propane on it. - -We make eye contact. We nod politely. I walk past him. "Hey! -Excuse me sir!" he says. - -"Yes?" I say, stopping and turning. - -"Have you seen," the guy says rapidly, "a black guy, about 6'7", -scars on both his cheeks like this--" he gestures-- "wears a -black baseball cap on backwards, wandering around here anyplace?" - -"Sounds like I don't much WANT to meet him," I say. - -"He took my wallet," says my new acquaintance. -"Took it this morning. Y'know, some people would be -SCARED of a guy like that. But I'm not scared. -I'm from Chicago. I'm gonna hunt him down. -We do things like that in Chicago." - -"Yeah?" - -"I went to the cops and now he's got an APB out on his ass," -he says with satisfaction. "You run into him, you let me know." - -"Okay," I say. "What is your name, sir?" - -"Stanley. . . ." - -"And how can I reach you?" - -"Oh," Stanley says, in the same rapid voice, -"you don't have to reach, uh, me. -You can just call the cops. Go straight to the cops." -He reaches into a pocket and pulls out a greasy piece of pasteboard. -"See, here's my report on him." - -I look. The "report," the size of an index card, is labelled PRO-ACT: -Phoenix Residents Opposing Active Crime Threat. . . . or is it -Organized Against Crime Threat? In the darkening street it's hard -to read. Some kind of vigilante group? Neighborhood watch? -I feel very puzzled. - -"Are you a police officer, sir?" - -He smiles, seems very pleased by the question. - -"No," he says. - -"But you are a `Phoenix Resident?'" - -"Would you believe a homeless person," Stanley says. - -"Really? But what's with the. . . ." For the first time I take a close look -at Stanley's trolley. It's a rubber-wheeled thing of industrial metal, -but the device I had mistaken for a tank of propane is in fact a water-cooler. -Stanley also has an Army duffel-bag, stuffed tight as a sausage with clothing -or perhaps a tent, and, at the base of his trolley, a cardboard box and a -battered leather briefcase. - -"I see," I say, quite at a loss. For the first time I notice that Stanley -has a wallet. He has not lost his wallet at all. It is in his back pocket -and chained to his belt. It's not a new wallet. It seems to have seen -a lot of wear. - -"Well, you know how it is, brother," says Stanley. -Now that I know that he is homeless--A POSSIBLE -THREAT--my entire perception of him has changed -in an instant. His speech, which once seemed just -bright and enthusiastic, now seems to have a -dangerous tang of mania. "I have to do this!" -he assures me. "Track this guy down. . . . -It's a thing I do. . . you know. . .to keep myself together!" -He smiles, nods, lifts his trolley by its decaying rubber handgrips. - -"Gotta work together, y'know," Stanley booms, his face alight -with cheerfulness, "the police can't do everything!" -The gentlemen I met in my stroll in downtown Phoenix -are the only computer illiterates in this book. -To regard them as irrelevant, however, would be a grave mistake. - -As computerization spreads across society, the populace at large -is subjected to wave after wave of future shock. But, as a -necessary converse, the "computer community" itself is subjected -to wave after wave of incoming computer illiterates. -How will those currently enjoying America's digital bounty regard, -and treat, all this teeming refuse yearning to breathe free? -Will the electronic frontier be another Land of Opportunity-- -or an armed and monitored enclave, where the disenfranchised -snuggle on their cardboard at the locked doors of our houses of justice? - -Some people just don't get along with computers. They can't read. -They can't type. They just don't have it in their heads to master -arcane instructions in wirebound manuals. Somewhere, the process -of computerization of the populace will reach a limit. Some people-- -quite decent people maybe, who might have thrived in any other situation-- -will be left irretrievably outside the bounds. What's to be done with -these people, in the bright new shiny electroworld? How will they -be regarded, by the mouse-whizzing masters of cyberspace? With contempt? -Indifference? Fear? - -In retrospect, it astonishes me to realize how quickly poor Stanley -became a perceived threat. Surprise and fear are closely allied feelings. -And the world of computing is full of surprises. - -I met one character in the streets of Phoenix whose role in this book -is supremely and directly relevant. That personage was Stanley's giant -thieving scarred phantom. This phantasm is everywhere in this book. -He is the specter haunting cyberspace. - -Sometimes he's a maniac vandal ready to smash the phone system -for no sane reason at all. Sometimes he's a fascist fed, -coldly programming his mighty mainframes to destroy our Bill of Rights. -Sometimes he's a telco bureaucrat, covertly conspiring to register all modems -in the service of an Orwellian surveillance regime. Mostly, though, -this fearsome phantom is a "hacker." He's strange, he doesn't belong, -he's not authorized, he doesn't smell right, he's not keeping his proper place, -he's not one of us. The focus of fear is the hacker, for much the same -reasons that Stanley's fancied assailant is black. - -Stanley's demon can't go away, because he doesn't exist. -Despite singleminded and tremendous effort, he can't be arrested, -sued, jailed, or fired. The only constructive way to do ANYTHING -about him is to learn more about Stanley himself. This learning process -may be repellent, it may be ugly, it may involve grave elements of paranoiac -confusion, but it's necessary. Knowing Stanley requires something more -than class-crossing condescension. It requires more than steely -legal objectivity. It requires human compassion and sympathy. - -To know Stanley is to know his demon. If you know the other guy's demon, -then maybe you'll come to know some of your own. You'll be able to -separate reality from illusion. And then you won't do your cause, -and yourself, more harm than good. Like poor damned Stanley from Chicago did. - -# - -The Federal Computer Investigations Committee (FCIC) is the most important -and influential organization in the realm of American computer-crime. -Since the police of other countries have largely taken their computer-crime -cues from American methods, the FCIC might well be called the most important -computer crime group in the world. - -It is also, by federal standards, an organization of great unorthodoxy. -State and local investigators mix with federal agents. Lawyers, -financial auditors and computer-security programmers trade notes -with street cops. Industry vendors and telco security people show up -to explain their gadgetry and plead for protection and justice. -Private investigators, think-tank experts and industry pundits throw in -their two cents' worth. The FCIC is the antithesis of a formal bureaucracy. - -Members of the FCIC are obscurely proud of this fact; they recognize their -group as aberrant, but are entirely convinced that this, for them, -outright WEIRD behavior is nevertheless ABSOLUTELY NECESSARY -to get their jobs done. - -FCIC regulars --from the Secret Service, the FBI, the IRS, -the Department of Labor, the offices of federal attorneys, -state police, the Air Force, from military intelligence-- -often attend meetings, held hither and thither across the country, -at their own expense. The FCIC doesn't get grants. It doesn't -charge membership fees. It doesn't have a boss. It has no headquarters-- -just a mail drop in Washington DC, at the Fraud Division of the Secret Service. -It doesn't have a budget. It doesn't have schedules. It meets three times -a year--sort of. Sometimes it issues publications, but the FCIC -has no regular publisher, no treasurer, not even a secretary. -There are no minutes of FCIC meetings. Non-federal people are considered -"non-voting members," but there's not much in the way of elections. -There are no badges, lapel pins or certificates of membership. -Everyone is on a first-name basis. There are about forty of them. -Nobody knows how many, exactly. People come, people go-- -sometimes people "go" formally but still hang around anyway. -Nobody has ever exactly figured out what "membership" of this -"Committee" actually entails. - -Strange as this may seem to some, to anyone familiar with the social world -of computing, the "organization" of the FCIC is very recognizable. - -For years now, economists and management theorists have speculated -that the tidal wave of the information revolution would destroy rigid, -pyramidal bureaucracies, where everything is top-down and -centrally controlled. Highly trained "employees" would take on -much greater autonomy, being self-starting, and self-motivating, -moving from place to place, task to task, with great speed and fluidity. -"Ad-hocracy" would rule, with groups of people spontaneously knitting -together across organizational lines, tackling the problem at hand, -applying intense computer-aided expertise to it, and then vanishing -whence they came. - -This is more or less what has actually happened in the world of -federal computer investigation. With the conspicuous exception -of the phone companies, which are after all over a hundred years old, -practically EVERY organization that plays any important role in this book -functions just like the FCIC. The Chicago Task Force, the Arizona -Racketeering Unit, the Legion of Doom, the Phrack crowd, the -Electronic Frontier Foundation--they ALL look and act like "tiger teams" -or "user's groups." They are all electronic ad-hocracies leaping up -spontaneously to attempt to meet a need. - -Some are police. Some are, by strict definition, criminals. -Some are political interest-groups. But every single group -has that same quality of apparent spontaneity--"Hey, gang! -My uncle's got a barn--let's put on a show!" - -Every one of these groups is embarrassed by this "amateurism," -and, for the sake of their public image in a world of non-computer people, -they all attempt to look as stern and formal and impressive as possible. -These electronic frontier-dwellers resemble groups of nineteenth-century -pioneers hankering after the respectability of statehood. -There are however, two crucial differences in the historical experience -of these "pioneers" of the nineteeth and twenty-first centuries. - -First, powerful information technology DOES play into the hands of small, -fluid, loosely organized groups. There have always been "pioneers," -"hobbyists," "amateurs," "dilettantes," "volunteers," "movements," -"users' groups" and "blue-ribbon panels of experts" around. -But a group of this kind--when technically equipped to ship -huge amounts of specialized information, at lightning speed, -to its members, to government, and to the press--is simply -a different kind of animal. It's like the difference between -an eel and an electric eel. - -The second crucial change is that American society is currently -in a state approaching permanent technological revolution. -In the world of computers particularly, it is practically impossible -to EVER stop being a "pioneer," unless you either drop dead or -deliberately jump off the bus. The scene has never slowed down -enough to become well-institutionalized. And after twenty, thirty, -forty years the "computer revolution" continues to spread, -to permeate new corners of society. Anything that really works -is already obsolete. - -If you spend your entire working life as a "pioneer," the word "pioneer" -begins to lose its meaning. Your way of life looks less and less like -an introduction to something else" more stable and organized, -and more and more like JUST THE WAY THINGS ARE. A "permanent revolution" -is really a contradiction in terms. If "turmoil" lasts long enough, -it simply becomes A NEW KIND OF SOCIETY--still the same game of history, -but new players, new rules. - -Apply this to the world of late twentieth-century law enforcement, -and the implications are novel and puzzling indeed. Any bureaucratic -rulebook you write about computer-crime will be flawed when you write it, -and almost an antique by the time it sees print. The fluidity and fast -reactions of the FCIC give them a great advantage in this regard, -which explains their success. Even with the best will in the world -(which it does not, in fact, possess) it is impossible for an organization -the size of the U.S. Federal Bureau of Investigation to get up to speed -on the theory and practice of computer crime. If they tried to train all -their agents to do this, it would be SUICIDAL, as they would NEVER BE ABLE -TO DO ANYTHING ELSE. - -The FBI does try to train its agents in the basics of electronic crime, -at their base in Quantico, Virginia. And the Secret Service, along with -many other law enforcement groups, runs quite successful and well-attended -training courses on wire fraud, business crime, and computer intrusion -at the Federal Law Enforcement Training Center (FLETC, pronounced "fletsy") -in Glynco, Georgia. But the best efforts of these bureaucracies does not -remove the absolute need for a "cutting-edge mess" like the FCIC. - -For you see--the members of FCIC ARE the trainers of the rest -of law enforcement. Practically and literally speaking, -they are the Glynco computer-crime faculty by another name. -If the FCIC went over a cliff on a bus, the U.S. law enforcement -community would be rendered deaf dumb and blind in the world -of computer crime, and would swiftly feel a desperate need -to reinvent them. And this is no time to go starting from scratch. - -On June 11, 1991, I once again arrived in Phoenix, Arizona, -for the latest meeting of the Federal Computer Investigations Committee. -This was more or less the twentieth meeting of this stellar group. -The count was uncertain, since nobody could figure out whether to -include the meetings of "the Colluquy," which is what the FCIC -was called in the mid-1980s before it had even managed to obtain -the dignity of its own acronym. - -Since my last visit to Arizona, in May, the local AzScam bribery scandal -had resolved itself in a general muddle of humiliation. The Phoenix chief -of police, whose agents had videotaped nine state legislators up to no good, -had resigned his office in a tussle with the Phoenix city council over -the propriety of his undercover operations. - -The Phoenix Chief could now join Gail Thackeray and eleven of her closest -associates in the shared experience of politically motivated unemployment. -As of June, resignations were still continuing at the Arizona Attorney -General's office, which could be interpreted as either a New Broom -Sweeping Clean or a Night of the Long Knives Part II, depending on -your point of view. - -The meeting of FCIC was held at the Scottsdale Hilton Resort. -Scottsdale is a wealthy suburb of Phoenix, known as "Scottsdull" -to scoffing local trendies, but well-equipped with posh shopping-malls -and manicured lawns, while conspicuously undersupplied with homeless derelicts. -The Scottsdale Hilton Resort was a sprawling hotel in postmodern -crypto-Southwestern style. It featured a "mission bell tower" -plated in turquoise tile and vaguely resembling a Saudi minaret. - -Inside it was all barbarically striped Santa Fe Style decor. -There was a health spa downstairs and a large oddly-shaped -pool in the patio. A poolside umbrella-stand offered Ben and Jerry's -politically correct Peace Pops. - -I registered as a member of FCIC, attaining a handy discount rate, -then went in search of the Feds. Sure enough, at the back of the -hotel grounds came the unmistakable sound of Gail Thackeray -holding forth. - -Since I had also attended the Computers Freedom and Privacy conference -(about which more later), this was the second time I had seen Thackeray -in a group of her law enforcement colleagues. Once again I was struck -by how simply pleased they seemed to see her. It was natural that she'd -get SOME attention, as Gail was one of two women in a group of some thirty men; -but there was a lot more to it than that. - -Gail Thackeray personifies the social glue of the FCIC. They could give -a damn about her losing her job with the Attorney General. They were sorry -about it, of course, but hell, they'd all lost jobs. If they were the kind -of guys who liked steady boring jobs, they would never have gotten into -computer work in the first place. - -I wandered into her circle and was immediately introduced to five strangers. -The conditions of my visit at FCIC were reviewed. I would not quote -anyone directly. I would not tie opinions expressed to the agencies -of the attendees. I would not (a purely hypothetical example) -report the conversation of a guy from the Secret Service talking -quite civilly to a guy from the FBI, as these two agencies NEVER -talk to each other, and the IRS (also present, also hypothetical) -NEVER TALKS TO ANYBODY. - -Worse yet, I was forbidden to attend the first conference. And I didn't. -I have no idea what the FCIC was up to behind closed doors that afternoon. -I rather suspect that they were engaging in a frank and thorough confession -of their errors, goof-ups and blunders, as this has been a feature of every -FCIC meeting since their legendary Memphis beer-bust of 1986. Perhaps the -single greatest attraction of FCIC is that it is a place where you can go, -let your hair down, and completely level with people who actually comprehend -what you are talking about. Not only do they understand you, but they -REALLY PAY ATTENTION, they are GRATEFUL FOR YOUR INSIGHTS, and they -FORGIVE YOU, which in nine cases out of ten is something even your -boss can't do, because as soon as you start talking "ROM," "BBS," -or "T-1 trunk," his eyes glaze over. - -I had nothing much to do that afternoon. The FCIC were beavering away -in their conference room. Doors were firmly closed, windows too dark -to peer through. I wondered what a real hacker, a computer intruder, -would do at a meeting like this. - -The answer came at once. He would "trash" the place. Not reduce the place -to trash in some orgy of vandalism; that's not the use of the term in the -hacker milieu. No, he would quietly EMPTY THE TRASH BASKETS and silently -raid any valuable data indiscreetly thrown away. - -Journalists have been known to do this. (Journalists hunting information -have been known to do almost every single unethical thing that hackers -have ever done. They also throw in a few awful techniques all their own.) -The legality of `trashing' is somewhat dubious but it is not in fact -flagrantly illegal. It was, however, absurd to contemplate trashing the FCIC. -These people knew all about trashing. I wouldn't last fifteen seconds. - -The idea sounded interesting, though. I'd been hearing a lot about -the practice lately. On the spur of the moment, I decided I would try -trashing the office ACROSS THE HALL from the FCIC, an area which had -nothing to do with the investigators. - -The office was tiny; six chairs, a table. . . . Nevertheless, it was open, -so I dug around in its plastic trash can. - -To my utter astonishment, I came up with the torn scraps of a SPRINT -long-distance phone bill. More digging produced a bank statement -and the scraps of a hand-written letter, along with gum, cigarette ashes, -candy wrappers and a day-old-issue of USA TODAY. - -The trash went back in its receptacle while the scraps of data went into -my travel bag. I detoured through the hotel souvenir shop for some -Scotch tape and went up to my room. - -Coincidence or not, it was quite true. Some poor soul had, in fact, -thrown a SPRINT bill into the hotel's trash. Date May 1991, -total amount due: $252.36. Not a business phone, either, -but a residential bill, in the name of someone called Evelyn -(not her real name). Evelyn's records showed a ## PAST DUE BILL ##! -Here was her nine-digit account ID. Here was a stern computer-printed warning: - -"TREAT YOUR FONCARD AS YOU WOULD ANY CREDIT CARD. TO SECURE AGAINST FRAUD, -NEVER GIVE YOUR FONCARD NUMBER OVER THE PHONE UNLESS YOU INITIATED THE CALL. -IF YOU RECEIVE SUSPICIOUS CALLS PLEASE NOTIFY CUSTOMER SERVICE IMMEDIATELY!" - -I examined my watch. Still plenty of time left for the FCIC to carry on. -I sorted out the scraps of Evelyn's SPRINT bill and re-assembled them with -fresh Scotch tape. Here was her ten-digit FONCARD number. Didn't seem -to have the ID number necessary to cause real fraud trouble. - -I did, however, have Evelyn's home phone number. And the phone numbers -for a whole crowd of Evelyn's long-distance friends and acquaintances. -In San Diego, Folsom, Redondo, Las Vegas, La Jolla, Topeka, and Northampton -Massachusetts. Even somebody in Australia! - -I examined other documents. Here was a bank statement. It was Evelyn's -IRA account down at a bank in San Mateo California (total balance $1877.20). -Here was a charge-card bill for $382.64. She was paying it off bit by bit. - -Driven by motives that were completely unethical and prurient, -I now examined the handwritten notes. They had been torn fairly -thoroughly, so much so that it took me almost an entire five minutes -to reassemble them. - -They were drafts of a love letter. They had been written on -the lined stationery of Evelyn's employer, a biomedical company. -Probably written at work when she should have been doing something else. - -"Dear Bob," (not his real name) "I guess in everyone's life there comes -a time when hard decisions have to be made, and this is a difficult one -for me--very upsetting. Since you haven't called me, and I don't understand -why, I can only surmise it's because you don't want to. I thought I would -have heard from you Friday. I did have a few unusual problems with my phone -and possibly you tried, I hope so. - -"Robert, you asked me to `let go'. . . ." - -The first note ended. UNUSUAL PROBLEMS WITH HER PHONE? -I looked swiftly at the next note. - -"Bob, not hearing from you for the whole weekend has left me very perplexed. . . ." - -Next draft. - -"Dear Bob, there is so much I don't understand right now, and I wish I did. -I wish I could talk to you, but for some unknown reason you have elected not -to call--this is so difficult for me to understand. . . ." - -She tried again. - -"Bob, Since I have always held you in such high esteem, I had every hope that -we could remain good friends, but now one essential ingredient is missing-- -respect. Your ability to discard people when their purpose is served is -appalling to me. The kindest thing you could do for me now is to leave me -alone. You are no longer welcome in my heart or home. . . ." - -Try again. - -"Bob, I wrote a very factual note to you to say how much respect I had lost -for you, by the way you treat people, me in particular, so uncaring and cold. -The kindest thing you can do for me is to leave me alone entirely, -as you are no longer welcome in my heart or home. I would appreciate it -if you could retire your debt to me as soon as possible--I wish no link -to you in any way. Sincerely, Evelyn." - -Good heavens, I thought, the bastard actually owes her money! -I turned to the next page. - -"Bob: very simple. GOODBYE! No more mind games--no more fascination-- -no more coldness--no more respect for you! It's over--Finis. Evie" - -There were two versions of the final brushoff letter, but they read about -the same. Maybe she hadn't sent it. The final item in my illicit and -shameful booty was an envelope addressed to "Bob" at his home address, -but it had no stamp on it and it hadn't been mailed. - -Maybe she'd just been blowing off steam because her rascal boyfriend -had neglected to call her one weekend. Big deal. Maybe they'd kissed -and made up, maybe she and Bob were down at Pop's Chocolate Shop now, -sharing a malted. Sure. - -Easy to find out. All I had to do was call Evelyn up. With a half-clever -story and enough brass-plated gall I could probably trick the truth out of her. -Phone-phreaks and hackers deceive people over the phone all the time. -It's called "social engineering." Social engineering is a very common practice -in the underground, and almost magically effective. Human beings are almost -always the weakest link in computer security. The simplest way to learn -Things You Are Not Meant To Know is simply to call up and exploit the -knowledgeable people. With social engineering, you use the bits of specialized -knowledge you already have as a key, to manipulate people into believing -that you are legitimate. You can then coax, flatter, or frighten them into -revealing almost anything you want to know. Deceiving people (especially -over the phone) is easy and fun. Exploiting their gullibility is very -gratifying; it makes you feel very superior to them. - -If I'd been a malicious hacker on a trashing raid, I would now have Evelyn -very much in my power. Given all this inside data, it wouldn't take much -effort at all to invent a convincing lie. If I were ruthless enough, -and jaded enough, and clever enough, this momentary indiscretion of hers-- -maybe committed in tears, who knows--could cause her a whole world of -confusion and grief. - -I didn't even have to have a MALICIOUS motive. Maybe I'd be "on her side," -and call up Bob instead, and anonymously threaten to break both his kneecaps -if he didn't take Evelyn out for a steak dinner pronto. It was still -profoundly NONE OF MY BUSINESS. To have gotten this knowledge at all -was a sordid act and to use it would be to inflict a sordid injury. - -To do all these awful things would require exactly zero high-tech expertise. -All it would take was the willingness to do it and a certain amount -of bent imagination. - -I went back downstairs. The hard-working FCIC, who had labored forty-five -minutes over their schedule, were through for the day, and adjourned to the -hotel bar. We all had a beer. - -I had a chat with a guy about "Isis," or rather IACIS, -the International Association of Computer Investigation Specialists. -They're into "computer forensics," the techniques of picking computer- -systems apart without destroying vital evidence. IACIS, currently run -out of Oregon, is comprised of investigators in the U.S., Canada, Taiwan -and Ireland. "Taiwan and Ireland?" I said. Are TAIWAN and IRELAND -really in the forefront of this stuff? Well not exactly, my informant -admitted. They just happen to have been the first ones to have caught -on by word of mouth. Still, the international angle counts, because this -is obviously an international problem. Phone-lines go everywhere. - -There was a Mountie here from the Royal Canadian Mounted Police. -He seemed to be having quite a good time. Nobody had flung this -Canadian out because he might pose a foreign security risk. -These are cyberspace cops. They still worry a lot about "jurisdictions," -but mere geography is the least of their troubles. - -NASA had failed to show. NASA suffers a lot from computer intrusions, -in particular from Australian raiders and a well-trumpeted Chaos -Computer Club case, and in 1990 there was a brief press flurry -when it was revealed that one of NASA's Houston branch-exchanges -had been systematically ripped off by a gang of phone-phreaks. -But the NASA guys had had their funding cut. They were stripping everything. - -Air Force OSI, its Office of Special Investigations, is the ONLY federal -entity dedicated full-time to computer security. They'd been expected -to show up in force, but some of them had cancelled--a Pentagon budget pinch. - -As the empties piled up, the guys began joshing around and telling war-stories. -"These are cops," Thackeray said tolerantly. "If they're not talking shop -they talk about women and beer." - -I heard the story about the guy who, asked for "a copy" of a computer disk, -PHOTOCOPIED THE LABEL ON IT. He put the floppy disk onto the glass plate -of a photocopier. The blast of static when the copier worked completely -erased all the real information on the disk. - -Some other poor souls threw a whole bag of confiscated diskettes -into the squad-car trunk next to the police radio. The powerful radio -signal blasted them, too. - -We heard a bit about Dave Geneson, the first computer prosecutor, -a mainframe-runner in Dade County, turned lawyer. Dave Geneson -was one guy who had hit the ground running, a signal virtue -in making the transition to computer-crime. It was generally -agreed that it was easier to learn the world of computers first, -then police or prosecutorial work. You could take certain computer -people and train 'em to successful police work--but of course they -had to have the COP MENTALITY. They had to have street smarts. -Patience. Persistence. And discretion. You've got to make sure -they're not hot-shots, show-offs, "cowboys." - -Most of the folks in the bar had backgrounds in military intelligence, -or drugs, or homicide. It was rudely opined that "military intelligence" -was a contradiction in terms, while even the grisly world of homicide -was considered cleaner than drug enforcement. One guy had been 'way -undercover doing dope-work in Europe for four years straight. -"I'm almost recovered now," he said deadpan, with the acid black humor -that is pure cop. "Hey, now I can say FUCKER without putting MOTHER -in front of it." - -"In the cop world," another guy said earnestly, "everything is good and bad, -black and white. In the computer world everything is gray." - -One guy--a founder of the FCIC, who'd been with the group -since it was just the Colluquy--described his own introduction -to the field. He'd been a Washington DC homicide guy called in -on a "hacker" case. From the word "hacker," he naturally assumed -he was on the trail of a knife-wielding marauder, and went to the -computer center expecting blood and a body. When he finally figured -out what was happening there (after loudly demanding, in vain, -that the programmers "speak English"), he called headquarters -and told them he was clueless about computers. They told him nobody -else knew diddly either, and to get the hell back to work. - -So, he said, he had proceeded by comparisons. By analogy. By metaphor. -"Somebody broke in to your computer, huh?" Breaking and entering; -I can understand that. How'd he get in? "Over the phone-lines." -Harassing phone-calls, I can understand that! What we need here -is a tap and a trace! - -It worked. It was better than nothing. And it worked a lot faster -when he got hold of another cop who'd done something similar. -And then the two of them got another, and another, and pretty soon -the Colluquy was a happening thing. It helped a lot that everybody -seemed to know Carlton Fitzpatrick, the data-processing trainer in Glynco. - -The ice broke big-time in Memphis in '86. The Colluquy had attracted -a bunch of new guys--Secret Service, FBI, military, other feds, heavy guys. -Nobody wanted to tell anybody anything. They suspected that if word got back -to the home office they'd all be fired. They passed an uncomfortably -guarded afternoon. - -The formalities got them nowhere. But after the formal session was over, -the organizers brought in a case of beer. As soon as the participants -knocked it off with the bureaucratic ranks and turf-fighting, everything -changed. "I bared my soul," one veteran reminisced proudly. By nightfall -they were building pyramids of empty beer-cans and doing everything -but composing a team fight song. - -FCIC were not the only computer-crime people around. There was DATTA -(District Attorneys' Technology Theft Association), though they mostly -specialized in chip theft, intellectual property, and black-market cases. -There was HTCIA (High Tech Computer Investigators Association), -also out in Silicon Valley, a year older than FCIC and featuring -brilliant people like Donald Ingraham. There was LEETAC -(Law Enforcement Electronic Technology Assistance Committee) -in Florida, and computer-crime units in Illinois and Maryland -and Texas and Ohio and Colorado and Pennsylvania. But these were -local groups. FCIC were the first to really network nationally -and on a federal level. - -FCIC people live on the phone lines. Not on bulletin board systems-- -they know very well what boards are, and they know that boards aren't secure. -Everyone in the FCIC has a voice-phone bill like you wouldn't believe. -FCIC people have been tight with the telco people for a long time. -Telephone cyberspace is their native habitat. - -FCIC has three basic sub-tribes: the trainers, the security people, -and the investigators. That's why it's called an "Investigations -Committee" with no mention of the term "computer-crime"--the dreaded -"C-word." FCIC, officially, is "an association of agencies rather -than individuals;" unofficially, this field is small enough that -the influence of individuals and individual expertise is paramount. -Attendance is by invitation only, and most everyone in FCIC considers -himself a prophet without honor in his own house. - -Again and again I heard this, with different terms but identical -sentiments. "I'd been sitting in the wilderness talking to myself." -"I was totally isolated." "I was desperate." "FCIC is the best -thing there is about computer crime in America." "FCIC is what -really works." "This is where you hear real people telling you -what's really happening out there, not just lawyers picking nits." -"We taught each other everything we knew." - -The sincerity of these statements convinces me that this is true. -FCIC is the real thing and it is invaluable. It's also very sharply -at odds with the rest of the traditions and power structure -in American law enforcement. There probably hasn't been anything -around as loose and go-getting as the FCIC since the start of the -U.S. Secret Service in the 1860s. FCIC people are living like -twenty-first-century people in a twentieth-century environment, -and while there's a great deal to be said for that, there's also -a great deal to be said against it, and those against it happen -to control the budgets. - -I listened to two FCIC guys from Jersey compare life histories. -One of them had been a biker in a fairly heavy-duty gang in the 1960s. -"Oh, did you know so-and-so?" said the other guy from Jersey. -"Big guy, heavyset?" - -"Yeah, I knew him." - -"Yeah, he was one of ours. He was our plant in the gang." - -"Really? Wow! Yeah, I knew him. Helluva guy." - -Thackeray reminisced at length about being tear-gassed blind -in the November 1969 antiwar protests in Washington Circle, -covering them for her college paper. "Oh yeah, I was there," -said another cop. "Glad to hear that tear gas hit somethin'. -Haw haw haw." He'd been so blind himself, he confessed, -that later that day he'd arrested a small tree. - -FCIC are an odd group, sifted out by coincidence and necessity, -and turned into a new kind of cop. There are a lot of specialized -cops in the world--your bunco guys, your drug guys, your tax guys, -but the only group that matches FCIC for sheer isolation are probably -the child-pornography people. Because they both deal with conspirators -who are desperate to exchange forbidden data and also desperate to hide; -and because nobody else in law enforcement even wants to hear about it. - -FCIC people tend to change jobs a lot. They tend not to get the equipment -and training they want and need. And they tend to get sued quite often. - -As the night wore on and a band set up in the bar, the talk grew darker. -Nothing ever gets done in government, someone opined, until there's -a DISASTER. Computing disasters are awful, but there's no denying -that they greatly help the credibility of FCIC people. The Internet Worm, -for instance. "For years we'd been warning about that--but it's nothing -compared to what's coming." They expect horrors, these people. -They know that nothing will really get done until there is a horror. - -# - -Next day we heard an extensive briefing from a guy who'd been a computer cop, -gotten into hot water with an Arizona city council, and now installed -computer networks for a living (at a considerable rise in pay). -He talked about pulling fiber-optic networks apart. - -Even a single computer, with enough peripherals, is a literal -"network"--a bunch of machines all cabled together, generally -with a complexity that puts stereo units to shame. FCIC people -invent and publicize methods of seizing computers and maintaining -their evidence. Simple things, sometimes, but vital rules of thumb -for street cops, who nowadays often stumble across a busy computer -in the midst of a drug investigation or a white-collar bust. -For instance: Photograph the system before you touch it. -Label the ends of all the cables before you detach anything. -"Park" the heads on the disk drives before you move them. -Get the diskettes. Don't put the diskettes in magnetic fields. -Don't write on diskettes with ballpoint pens. Get the manuals. -Get the printouts. Get the handwritten notes. Copy data before -you look at it, and then examine the copy instead of the original. - -Now our lecturer distributed copied diagrams of a typical LAN -or "Local Area Network", which happened to be out of Connecticut. -ONE HUNDRED AND FIFTY-NINE desktop computers, each with its own -peripherals. Three "file servers." Five "star couplers" -each with thirty-two ports. One sixteen-port coupler -off in the corner office. All these machines talking to each other, -distributing electronic mail, distributing software, distributing, -quite possibly, criminal evidence. All linked by high-capacity -fiber-optic cable. A bad guy--cops talk a about "bad guys" ---might be lurking on PC #47 lot or #123 and distributing -his ill doings onto some dupe's "personal" machine in -another office--or another floor--or, quite possibly, -two or three miles away! Or, conceivably, the evidence might -be "data-striped"--split up into meaningless slivers stored, -one by one, on a whole crowd of different disk drives. - -The lecturer challenged us for solutions. I for one was utterly clueless. -As far as I could figure, the Cossacks were at the gate; there were probably -more disks in this single building than were seized during the entirety -of Operation Sundevil. - -"Inside informant," somebody said. Right. There's always the human angle, -something easy to forget when contemplating the arcane recesses of high -technology. Cops are skilled at getting people to talk, and computer people, -given a chair and some sustained attention, will talk about their computers -till their throats go raw. There's a case on record of a single question-- -"How'd you do it?"--eliciting a forty-five-minute videotaped confession -from a computer criminal who not only completely incriminated himself -but drew helpful diagrams. - -Computer people talk. Hackers BRAG. Phone-phreaks -talk PATHOLOGICALLY--why else are they stealing phone-codes, -if not to natter for ten hours straight to their friends -on an opposite seaboard? Computer-literate people do -in fact possess an arsenal of nifty gadgets and techniques -that would allow them to conceal all kinds of exotic skullduggery, -and if they could only SHUT UP about it, they could probably -get away with all manner of amazing information-crimes. -But that's just not how it works--or at least, -that's not how it's worked SO FAR. - -Most every phone-phreak ever busted has swiftly implicated his mentors, -his disciples, and his friends. Most every white-collar computer-criminal, -smugly convinced that his clever scheme is bulletproof, swiftly learns -otherwise when, for the first time in his life, an actual no-kidding -policeman leans over, grabs the front of his shirt, looks him right -in the eye and says: "All right, ASSHOLE--you and me are going downtown!" -All the hardware in the world will not insulate your nerves from -these actual real-life sensations of terror and guilt. - -Cops know ways to get from point A to point Z without thumbing -through every letter in some smart-ass bad-guy's alphabet. -Cops know how to cut to the chase. Cops know a lot of things -other people don't know. - -Hackers know a lot of things other people don't know, too. -Hackers know, for instance, how to sneak into your computer -through the phone-lines. But cops can show up RIGHT ON YOUR DOORSTEP -and carry off YOU and your computer in separate steel boxes. -A cop interested in hackers can grab them and grill them. -A hacker interested in cops has to depend on hearsay, -underground legends, and what cops are willing to publicly reveal. -And the Secret Service didn't get named "the SECRET Service" -because they blab a lot. - -Some people, our lecturer informed us, were under the mistaken -impression that it was "impossible" to tap a fiber-optic line. -Well, he announced, he and his son had just whipped up a -fiber-optic tap in his workshop at home. He passed it around -the audience, along with a circuit-covered LAN plug-in card -so we'd all recognize one if we saw it on a case. We all had a look. - -The tap was a classic "Goofy Prototype"--a thumb-length rounded -metal cylinder with a pair of plastic brackets on it. -From one end dangled three thin black cables, each of which ended -in a tiny black plastic cap. When you plucked the safety-cap -off the end of a cable, you could see the glass fiber-- -no thicker than a pinhole. - -Our lecturer informed us that the metal cylinder was a -"wavelength division multiplexer." Apparently, what one did -was to cut the fiber-optic cable, insert two of the legs into -the cut to complete the network again, and then read any passing data -on the line by hooking up the third leg to some kind of monitor. -Sounded simple enough. I wondered why nobody had thought of it before. -I also wondered whether this guy's son back at the workshop had any -teenage friends. - -We had a break. The guy sitting next to me was wearing a giveaway -baseball cap advertising the Uzi submachine gun. We had a desultory chat -about the merits of Uzis. Long a favorite of the Secret Service, -it seems Uzis went out of fashion with the advent of the Persian Gulf War, -our Arab allies taking some offense at Americans toting Israeli weapons. -Besides, I was informed by another expert, Uzis jam. The equivalent weapon -of choice today is the Heckler & Koch, manufactured in Germany. - -The guy with the Uzi cap was a forensic photographer. He also did a lot -of photographic surveillance work in computer crime cases. He used to, -that is, until the firings in Phoenix. He was now a private investigator and, -with his wife, ran a photography salon specializing in weddings and portrait -photos. At--one must repeat--a considerable rise in income. - -He was still FCIC. If you were FCIC, and you needed to talk -to an expert about forensic photography, well, there he was, -willing and able. If he hadn't shown up, people would have missed him. - -Our lecturer had raised the point that preliminary investigation -of a computer system is vital before any seizure is undertaken. -It's vital to understand how many machines are in there, what kinds -there are, what kind of operating system they use, how many people -use them, where the actual data itself is stored. To simply barge into -an office demanding "all the computers" is a recipe for swift disaster. - -This entails some discreet inquiries beforehand. In fact, what it -entails is basically undercover work. An intelligence operation. -SPYING, not to put too fine a point on it. - -In a chat after the lecture, I asked an attendee whether "trashing" might work. - -I received a swift briefing on the theory and practice of "trash covers." -Police "trash covers," like "mail covers" or like wiretaps, require the -agreement of a judge. This obtained, the "trashing" work of cops is just -like that of hackers, only more so and much better organized. So much so, -I was informed, that mobsters in Phoenix make extensive use of locked -garbage cans picked up by a specialty high-security trash company. - -In one case, a tiger team of Arizona cops had trashed a local residence -for four months. Every week they showed up on the municipal garbage truck, -disguised as garbagemen, and carried the contents of the suspect cans off -to a shade tree, where they combed through the garbage--a messy task, -especially considering that one of the occupants was undergoing -kidney dialysis. All useful documents were cleaned, dried and examined. -A discarded typewriter-ribbon was an especially valuable source of data, -as its long one-strike ribbon of film contained the contents of every -letter mailed out of the house. The letters were neatly retyped by -a police secretary equipped with a large desk-mounted magnifying glass. - -There is something weirdly disquieting about the whole subject of -"trashing"-- an unsuspected and indeed rather disgusting mode of -deep personal vulnerability. Things that we pass by every day, -that we take utterly for granted, can be exploited with so little work. -Once discovered, the knowledge of these vulnerabilities tend to spread. - -Take the lowly subject of MANHOLE COVERS. The humble manhole cover -reproduces many of the dilemmas of computer-security in miniature. -Manhole covers are, of course, technological artifacts, access-points -to our buried urban infrastructure. To the vast majority of us, -manhole covers are invisible. They are also vulnerable. For many years now, -the Secret Service has made a point of caulking manhole covers along all routes -of the Presidential motorcade. This is, of course, to deter terrorists from -leaping out of underground ambush or, more likely, planting remote-control -car-smashing bombs beneath the street. - -Lately, manhole covers have seen more and more criminal exploitation, -especially in New York City. Recently, a telco in New York City -discovered that a cable television service had been sneaking into -telco manholes and installing cable service alongside the phone-lines-- -WITHOUT PAYING ROYALTIES. New York companies have also suffered a -general plague of (a) underground copper cable theft; (b) dumping of garbage, -including toxic waste, and (c) hasty dumping of murder victims. - -Industry complaints reached the ears of an innovative New England -industrial-security company, and the result was a new product known -as "the Intimidator," a thick titanium-steel bolt with a precisely machined -head that requires a special device to unscrew. All these "keys" have registered -serial numbers kept on file with the manufacturer. There are now some -thousands of these "Intimidator" bolts being sunk into American pavements -wherever our President passes, like some macabre parody of strewn roses. -They are also spreading as fast as steel dandelions around US military bases -and many centers of private industry. - -Quite likely it has never occurred to you to peer under a manhole cover, -perhaps climb down and walk around down there with a flashlight, just to see -what it's like. Formally speaking, this might be trespassing, but if you -didn't hurt anything, and didn't make an absolute habit of it, nobody would -really care. The freedom to sneak under manholes was likely a freedom -you never intended to exercise. - -You now are rather less likely to have that freedom at all. -You may never even have missed it until you read about it here, -but if you're in New York City it's gone, and elsewhere it's likely going. -This is one of the things that crime, and the reaction to -crime, does to us. - -The tenor of the meeting now changed as the Electronic Frontier Foundation -arrived. The EFF, whose personnel and history will be examined in detail -in the next chapter, are a pioneering civil liberties group who arose in -direct response to the Hacker Crackdown of 1990. - -Now Mitchell Kapor, the Foundation's president, and Michael Godwin, -its chief attorney, were confronting federal law enforcement MANO A MANO -for the first time ever. Ever alert to the manifold uses of publicity, -Mitch Kapor and Mike Godwin had brought their own journalist in tow: -Robert Draper, from Austin, whose recent well-received book about -ROLLING STONE magazine was still on the stands. Draper was on assignment -for TEXAS MONTHLY. - -The Steve Jackson/EFF civil lawsuit against the Chicago Computer Fraud -and Abuse Task Force was a matter of considerable regional interest in Texas. -There were now two Austinite journalists here on the case. In fact, -counting Godwin (a former Austinite and former journalist) there were -three of us. Lunch was like Old Home Week. - -Later, I took Draper up to my hotel room. We had a long frank talk -about the case, networking earnestly like a miniature freelance-journo -version of the FCIC: privately confessing the numerous blunders -of journalists covering the story, and trying hard to figure out -who was who and what the hell was really going on out there. -I showed Draper everything I had dug out of the Hilton trashcan. -We pondered the ethics of "trashing" for a while, and agreed -that they were dismal. We also agreed that finding a SPRINT -bill on your first time out was a heck of a coincidence. - -First I'd "trashed"--and now, mere hours later, I'd bragged to someone else. -Having entered the lifestyle of hackerdom, I was now, unsurprisingly, -following its logic. Having discovered something remarkable through -a surreptitious action, I of course HAD to "brag," and to drag the passing -Draper into my iniquities. I felt I needed a witness. Otherwise nobody -would have believed what I'd discovered. . . . - -Back at the meeting, Thackeray cordially, if rather tentatively, -introduced Kapor and Godwin to her colleagues. Papers were distributed. -Kapor took center stage. The brilliant Bostonian high-tech entrepreneur, -normally the hawk in his own administration and quite an effective -public speaker, seemed visibly nervous, and frankly admitted as much. -He began by saying he consided computer-intrusion to be morally wrong, -and that the EFF was not a "hacker defense fund," despite what had appeared -in print. Kapor chatted a bit about the basic motivations of his group, -emphasizing their good faith and willingness to listen and seek common ground -with law enforcement--when, er, possible. - -Then, at Godwin's urging, Kapor suddenly remarked that EFF's own Internet -machine had been "hacked" recently, and that EFF did not consider -this incident amusing. - -After this surprising confession, things began to loosen up -quite rapidly. Soon Kapor was fielding questions, parrying objections, -challenging definitions, and juggling paradigms with something akin -to his usual gusto. - -Kapor seemed to score quite an effect with his shrewd and skeptical analysis -of the merits of telco "Caller-ID" services. (On this topic, FCIC and EFF -have never been at loggerheads, and have no particular established earthworks -to defend.) Caller-ID has generally been promoted as a privacy service -for consumers, a presentation Kapor described as a "smokescreen," -the real point of Caller-ID being to ALLOW CORPORATE CUSTOMERS TO BUILD -EXTENSIVE COMMERCIAL DATABASES ON EVERYBODY WHO PHONES OR FAXES THEM. -Clearly, few people in the room had considered this possibility, -except perhaps for two late-arrivals from US WEST RBOC security, -who chuckled nervously. - -Mike Godwin then made an extensive presentation on -"Civil Liberties Implications of Computer Searches and Seizures." -Now, at last, we were getting to the real nitty-gritty here, -real political horse-trading. The audience listened with close -attention, angry mutters rising occasionally: "He's trying to -teach us our jobs!" "We've been thinking about this for years! -We think about these issues every day!" "If I didn't seize the works, -I'd be sued by the guy's victims!" "I'm violating the law if I leave -ten thousand disks full of illegal PIRATED SOFTWARE and STOLEN CODES!" -"It's our job to make sure people don't trash the Constitution-- -we're the DEFENDERS of the Constitution!" "We seize stuff when -we know it will be forfeited anyway as restitution for the victim!" - -"If it's forfeitable, then don't get a search warrant, get a -forfeiture warrant," Godwin suggested coolly. He further remarked -that most suspects in computer crime don't WANT to see their computers -vanish out the door, headed God knew where, for who knows how long. -They might not mind a search, even an extensive search, but they want -their machines searched on-site. - -"Are they gonna feed us?" somebody asked sourly. - -"How about if you take copies of the data?" Godwin parried. - -"That'll never stand up in court." - -"Okay, you make copies, give THEM the copies, and take the originals." - -Hmmm. - -Godwin championed bulletin-board systems as repositories of First Amendment -protected free speech. He complained that federal computer-crime training -manuals gave boards a bad press, suggesting that they are hotbeds of crime -haunted by pedophiles and crooks, whereas the vast majority of the nation's -thousands of boards are completely innocuous, and nowhere near so -romantically suspicious. - -People who run boards violently resent it when their systems are seized, -and their dozens (or hundreds) of users look on in abject horror. -Their rights of free expression are cut short. Their right to associate -with other people is infringed. And their privacy is violated as their -private electronic mail becomes police property. - -Not a soul spoke up to defend the practice of seizing boards. -The issue passed in chastened silence. Legal principles aside-- -(and those principles cannot be settled without laws passed or -court precedents)--seizing bulletin boards has become public-relations -poison for American computer police. - -And anyway, it's not entirely necessary. If you're a cop, you can get 'most -everything you need from a pirate board, just by using an inside informant. -Plenty of vigilantes--well, CONCERNED CITIZENS--will inform police the moment -they see a pirate board hit their area (and will tell the police all about it, -in such technical detail, actually, that you kinda wish they'd shut up). -They will happily supply police with extensive downloads or printouts. -It's IMPOSSIBLE to keep this fluid electronic information out of the -hands of police. - -Some people in the electronic community become enraged at the prospect -of cops "monitoring" bulletin boards. This does have touchy aspects, -as Secret Service people in particular examine bulletin boards with -some regularity. But to expect electronic police to be deaf dumb -and blind in regard to this particular medium rather flies in the face -of common sense. Police watch television, listen to radio, read newspapers -and magazines; why should the new medium of boards be different? -Cops can exercise the same access to electronic information -as everybody else. As we have seen, quite a few computer -police maintain THEIR OWN bulletin boards, including anti-hacker -"sting" boards, which have generally proven quite effective. - -As a final clincher, their Mountie friends in Canada (and colleagues -in Ireland and Taiwan) don't have First Amendment or American -constitutional restrictions, but they do have phone lines, -and can call any bulletin board in America whenever they please. -The same technological determinants that play into the hands of hackers, -phone phreaks and software pirates can play into the hands of police. -"Technological determinants" don't have ANY human allegiances. -They're not black or white, or Establishment or Underground, -or pro-or-anti anything. - -Godwin complained at length about what he called "the Clever Hobbyist -hypothesis" --the assumption that the "hacker" you're busting is clearly -a technical genius, and must therefore by searched with extreme thoroughness. -So: from the law's point of view, why risk missing anything? Take the works. -Take the guy's computer. Take his books. Take his notebooks. -Take the electronic drafts of his love letters. Take his Walkman. -Take his wife's computer. Take his dad's computer. Take his kid -sister's computer. Take his employer's computer. Take his compact disks-- -they MIGHT be CD-ROM disks, cunningly disguised as pop music. -Take his laser printer--he might have hidden something vital in the -printer's 5meg of memory. Take his software manuals and hardware -documentation. Take his science-fiction novels and his simulation- -gaming books. Take his Nintendo Game-Boy and his Pac-Man arcade game. -Take his answering machine, take his telephone out of the wall. -Take anything remotely suspicious. - -Godwin pointed out that most "hackers" are not, in fact, clever -genius hobbyists. Quite a few are crooks and grifters who don't -have much in the way of technical sophistication; just some rule-of-thumb -rip-off techniques. The same goes for most fifteen-year-olds who've -downloaded a code-scanning program from a pirate board. There's no -real need to seize everything in sight. It doesn't require an entire -computer system and ten thousand disks to prove a case in court. - -What if the computer is the instrumentality of a crime? someone demanded. - -Godwin admitted quietly that the doctrine of seizing the instrumentality -of a crime was pretty well established in the American legal system. - -The meeting broke up. Godwin and Kapor had to leave. Kapor was testifying -next morning before the Massachusetts Department Of Public Utility, -about ISDN narrowband wide-area networking. - -As soon as they were gone, Thackeray seemed elated. -She had taken a great risk with this. Her colleagues had not, -in fact, torn Kapor and Godwin's heads off. She was very proud of them, -and told them so. - -"Did you hear what Godwin said about INSTRUMENTALITY OF A CRIME?" -she exulted, to nobody in particular. "Wow, that means -MITCH ISN'T GOING TO SUE ME." - -# - -America's computer police are an interesting group. -As a social phenomenon they are far more interesting, -and far more important, than teenage phone phreaks -and computer hackers. First, they're older and wiser; -not dizzy hobbyists with leaky morals, but seasoned adult -professionals with all the responsibilities of public service. -And, unlike hackers, they possess not merely TECHNICAL -power alone, but heavy-duty legal and social authority. - -And, very interestingly, they are just as much at -sea in cyberspace as everyone else. They are not -happy about this. Police are authoritarian by nature, -and prefer to obey rules and precedents. (Even those police -who secretly enjoy a fast ride in rough territory will soberly -disclaim any "cowboy" attitude.) But in cyberspace there ARE -no rules and precedents. They are groundbreaking pioneers, -Cyberspace Rangers, whether they like it or not. - -In my opinion, any teenager enthralled by computers, -fascinated by the ins and outs of computer security, -and attracted by the lure of specialized forms of knowledge and power, -would do well to forget all about "hacking" and set his (or her) -sights on becoming a fed. Feds can trump hackers at almost every -single thing hackers do, including gathering intelligence, -undercover disguise, trashing, phone-tapping, building dossiers, -networking, and infiltrating computer systems--CRIMINAL computer systems. -Secret Service agents know more about phreaking, coding and carding -than most phreaks can find out in years, and when it comes to viruses, -break-ins, software bombs and trojan horses, Feds have direct access to red-hot -confidential information that is only vague rumor in the underground. - -And if it's an impressive public rep you're after, there are few people -in the world who can be so chillingly impressive as a well-trained, -well-armed United States Secret Service agent. - -Of course, a few personal sacrifices are necessary in order to obtain -that power and knowledge. First, you'll have the galling discipline -of belonging to a large organization; but the world of computer crime -is still so small, and so amazingly fast-moving, that it will remain -spectacularly fluid for years to come. The second sacrifice is that -you'll have to give up ripping people off. This is not a great loss. -Abstaining from the use of illegal drugs, also necessary, will be a boon -to your health. - -A career in computer security is not a bad choice for a young man -or woman today. The field will almost certainly expand drastically -in years to come. If you are a teenager today, by the time you -become a professional, the pioneers you have read about in this book -will be the grand old men and women of the field, swamped by their many -disciples and successors. Of course, some of them, like William P. Wood -of the 1865 Secret Service, may well be mangled in the whirring machinery -of legal controversy; but by the time you enter the computer-crime field, -it may have stabilized somewhat, while remaining entertainingly challenging. - -But you can't just have a badge. You have to win it. First, there's the -federal law enforcement training. And it's hard--it's a challenge. -A real challenge--not for wimps and rodents. - -Every Secret Service agent must complete gruelling courses at the -Federal Law Enforcement Training Center. (In fact, Secret Service -agents are periodically re-trained during their entire careers.) - -In order to get a glimpse of what this might be like, -I myself travelled to FLETC. - -# - -The Federal Law Enforcement Training Center is a 1500-acre facility -on Georgia's Atlantic coast. It's a milieu of marshgrass, seabirds, -damp, clinging sea-breezes, palmettos, mosquitos, and bats. -Until 1974, it was a Navy Air Base, and still features a working runway, -and some WWII vintage blockhouses and officers' quarters. -The Center has since benefitted by a forty-million-dollar retrofit, -but there's still enough forest and swamp on the facility for the -Border Patrol to put in tracking practice. - -As a town, "Glynco" scarcely exists. The nearest real town is Brunswick, -a few miles down Highway 17, where I stayed at the aptly named Marshview -Holiday Inn. I had Sunday dinner at a seafood restaurant called "Jinright's," -where I feasted on deep-fried alligator tail. This local favorite was -a heaped basket of bite-sized chunks of white, tender, almost fluffy -reptile meat, steaming in a peppered batter crust. Alligator makes -a culinary experience that's hard to forget, especially when liberally -basted with homemade cocktail sauce from a Jinright squeeze-bottle. - -The crowded clientele were tourists, fishermen, local black folks -in their Sunday best, and white Georgian locals who all seemed -to bear an uncanny resemblance to Georgia humorist Lewis Grizzard. - -The 2,400 students from 75 federal agencies who make up the FLETC -population scarcely seem to make a dent in the low-key local scene. -The students look like tourists, and the teachers seem to have taken -on much of the relaxed air of the Deep South. My host was Mr. Carlton -Fitzpatrick, the Program Coordinator of the Financial Fraud Institute. -Carlton Fitzpatrick is a mustached, sinewy, well-tanned Alabama native -somewhere near his late forties, with a fondness for chewing tobacco, -powerful computers, and salty, down-home homilies. We'd met before, -at FCIC in Arizona. - -The Financial Fraud Institute is one of the nine divisions at FLETC. -Besides Financial Fraud, there's Driver & Marine, Firearms, -and Physical Training. These are specialized pursuits. -There are also five general training divisions: Basic Training, -Operations, Enforcement Techniques, Legal Division, and Behavioral Science. - -Somewhere in this curriculum is everything necessary to turn green college -graduates into federal agents. First they're given ID cards. Then they get -the rather miserable-looking blue coveralls known as "smurf suits." -The trainees are assigned a barracks and a cafeteria, and immediately -set on FLETC's bone-grinding physical training routine. Besides the -obligatory daily jogging--(the trainers run up danger flags beside -the track when the humidity rises high enough to threaten heat stroke)-- -here's the Nautilus machines, the martial arts, the survival skills. . . . - -The eighteen federal agencies who maintain on-site academies at FLETC -employ a wide variety of specialized law enforcement units, some of them -rather arcane. There's Border Patrol, IRS Criminal Investigation Division, -Park Service, Fish and Wildlife, Customs, Immigration, Secret Service and -the Treasury's uniformed subdivisions. . . . If you're a federal cop -and you don't work for the FBI, you train at FLETC. This includes people -as apparently obscure as the agents of the Railroad Retirement Board -Inspector General. Or the Tennessee Valley Authority Police, -who are in fact federal police officers, and can and do arrest criminals -on the federal property of the Tennessee Valley Authority. - -And then there are the computer-crime people. All sorts, all backgrounds. -Mr. Fitzpatrick is not jealous of his specialized knowledge. Cops all over, -in every branch of service, may feel a need to learn what he can teach. -Backgrounds don't matter much. Fitzpatrick himself was originally a -Border Patrol veteran, then became a Border Patrol instructor at FLETC. -His Spanish is still fluent--but he found himself strangely fascinated -when the first computers showed up at the Training Center. Fitzpatrick -did have a background in electrical engineering, and though he never -considered himself a computer hacker, he somehow found himself writing -useful little programs for this new and promising gizmo. - -He began looking into the general subject of computers and crime, -reading Donn Parker's books and articles, keeping an ear cocked -for war stories, useful insights from the field, the up-and-coming -people of the local computer-crime and high-technology units. . . . -Soon he got a reputation around FLETC as the resident "computer expert," -and that reputation alone brought him more exposure, more experience-- -until one day he looked around, and sure enough he WAS a federal -computer-crime expert. - -In fact, this unassuming, genial man may be THE federal computer-crime expert. -There are plenty of very good computer people, and plenty of very good -federal investigators, but the area where these worlds of expertise overlap -is very slim. And Carlton Fitzpatrick has been right at the center of that -since 1985, the first year of the Colluquy, a group which owes much to -his influence. - -He seems quite at home in his modest, acoustic-tiled office, -with its Ansel Adams-style Western photographic art, a gold-framed -Senior Instructor Certificate, and a towering bookcase crammed with -three-ring binders with ominous titles such as Datapro Reports on -Information Security and CFCA Telecom Security '90. - -The phone rings every ten minutes; colleagues show up at the door -to chat about new developments in locksmithing or to shake their heads -over the latest dismal developments in the BCCI global banking scandal. - -Carlton Fitzpatrick is a fount of computer-crime war-stories, -related in an acerbic drawl. He tells me the colorful tale -of a hacker caught in California some years back. He'd been -raiding systems, typing code without a detectable break, -for twenty, twenty-four, thirty-six hours straight. Not just -logged on--TYPING. Investigators were baffled. Nobody -could do that. Didn't he have to go to the bathroom? -Was it some kind of automatic keyboard-whacking device -that could actually type code? - -A raid on the suspect's home revealed a situation of astonishing squalor. -The hacker turned out to be a Pakistani computer-science student who had -flunked out of a California university. He'd gone completely underground -as an illegal electronic immigrant, and was selling stolen phone-service -to stay alive. The place was not merely messy and dirty, but in a state -of psychotic disorder. Powered by some weird mix of culture shock, -computer addiction, and amphetamines, the suspect had in fact been sitting -in front of his computer for a day and a half straight, with snacks and -drugs at hand on the edge of his desk and a chamber-pot under his chair. - -Word about stuff like this gets around in the hacker-tracker community. - -Carlton Fitzpatrick takes me for a guided tour by car around the -FLETC grounds. One of our first sights is the biggest indoor -firing range in the world. There are federal trainees in there, -Fitzpatrick assures me politely, blasting away with a wide variety -of automatic weapons: Uzis, Glocks, AK-47s. . . . He's willing to -take me inside. I tell him I'm sure that's really interesting, -but I'd rather see his computers. Carlton Fitzpatrick seems quite -surprised and pleased. I'm apparently the first journalist he's ever -seen who has turned down the shooting gallery in favor of microchips. - -Our next stop is a favorite with touring Congressmen: the three-mile -long FLETC driving range. Here trainees of the Driver & Marine Division -are taught high-speed pursuit skills, setting and breaking road-blocks, -diplomatic security driving for VIP limousines. . . . A favorite FLETC -pastime is to strap a passing Senator into the passenger seat beside a -Driver & Marine trainer, hit a hundred miles an hour, then take it right into -"the skid-pan," a section of greased track where two tons of Detroit iron -can whip and spin like a hockey puck. - -Cars don't fare well at FLETC. First they're rifled again and again -for search practice. Then they do 25,000 miles of high-speed -pursuit training; they get about seventy miles per set -of steel-belted radials. Then it's off to the skid pan, -where sometimes they roll and tumble headlong in the grease. -When they're sufficiently grease-stained, dented, and creaky, -they're sent to the roadblock unit, where they're battered without pity. -And finally then they're sacrificed to the Bureau of Alcohol, -Tobacco and Firearms, whose trainees learn the ins and outs -of car-bomb work by blowing them into smoking wreckage. - -There's a railroad box-car on the FLETC grounds, and a large -grounded boat, and a propless plane; all training-grounds for searches. -The plane sits forlornly on a patch of weedy tarmac next to an eerie -blockhouse known as the "ninja compound," where anti-terrorism specialists -practice hostage rescues. As I gaze on this creepy paragon of modern -low-intensity warfare, my nerves are jangled by a sudden staccato outburst -of automatic weapons fire, somewhere in the woods to my right. -"Nine-millimeter," Fitzpatrick judges calmly. - -Even the eldritch ninja compound pales somewhat compared -to the truly surreal area known as "the raid-houses." -This is a street lined on both sides with nondescript -concrete-block houses with flat pebbled roofs. -They were once officers' quarters. Now they are training grounds. -The first one to our left, Fitzpatrick tells me, has been specially -adapted for computer search-and-seizure practice. Inside it has been -wired for video from top to bottom, with eighteen pan-and-tilt -remotely controlled videocams mounted on walls and in corners. -Every movement of the trainee agent is recorded live by teachers, -for later taped analysis. Wasted movements, hesitations, possibly lethal -tactical mistakes--all are gone over in detail. - -Perhaps the weirdest single aspect of this building is its front door, -scarred and scuffed all along the bottom, from the repeated impact, -day after day, of federal shoe-leather. - -Down at the far end of the row of raid-houses some people are practicing -a murder. We drive by slowly as some very young and rather nervous-looking -federal trainees interview a heavyset bald man on the raid-house lawn. -Dealing with murder takes a lot of practice; first you have to learn -to control your own instinctive disgust and panic, then you have to learn -to control the reactions of a nerve-shredded crowd of civilians, -some of whom may have just lost a loved one, some of whom may be murderers-- -quite possibly both at once. - -A dummy plays the corpse. The roles of the bereaved, the morbidly curious, -and the homicidal are played, for pay, by local Georgians: waitresses, -musicians, most anybody who needs to moonlight and can learn a script. -These people, some of whom are FLETC regulars year after year, -must surely have one of the strangest jobs in the world. - -Something about the scene: "normal" people in a weird situation, -standing around talking in bright Georgia sunshine, unsuccessfully -pretending that something dreadful has gone on, while a dummy lies -inside on faked bloodstains. . . . While behind this weird masquerade, -like a nested set of Russian dolls, are grim future realities of real death, -real violence, real murders of real people, that these young agents -will really investigate, many times during their careers. . . . -Over and over. . . . Will those anticipated murders look like this, -feel like this--not as "real" as these amateur actors are trying to -make it seem, but both as "real," and as numbingly unreal, as watching -fake people standing around on a fake lawn? Something about this scene -unhinges me. It seems nightmarish to me, Kafkaesque. I simply don't -know how to take it; my head is turned around; I don't know whether to laugh, -cry, or just shudder. - -When the tour is over, Carlton Fitzpatrick and I talk about computers. -For the first time cyberspace seems like quite a comfortable place. -It seems very real to me suddenly, a place where I know what I'm talking about, -a place I'm used to. It's real. "Real." Whatever. - -Carlton Fitzpatrick is the only person I've met in cyberspace circles -who is happy with his present equipment. He's got a 5 Meg RAM PC with -a 112 meg hard disk; a 660 meg's on the way. He's got a Compaq 386 desktop, -and a Zenith 386 laptop with 120 meg. Down the hall is a NEC Multi-Sync 2A -with a CD-ROM drive and a 9600 baud modem with four com-lines. -There's a training minicomputer, and a 10-meg local mini just for the Center, -and a lab-full of student PC clones and half-a-dozen Macs or so. -There's a Data General MV 2500 with 8 meg on board and a 370 meg disk. - -Fitzpatrick plans to run a UNIX board on the Data General when he's -finished beta-testing the software for it, which he wrote himself. -It'll have E-mail features, massive files on all manner of computer-crime -and investigation procedures, and will follow the computer-security -specifics of the Department of Defense "Orange Book." He thinks -it will be the biggest BBS in the federal government. - -Will it have Phrack on it? I ask wryly. - -Sure, he tells me. Phrack, TAP, Computer Underground Digest, -all that stuff. With proper disclaimers, of course. - -I ask him if he plans to be the sysop. Running a system that size is very -time-consuming, and Fitzpatrick teaches two three-hour courses every day. - -No, he says seriously, FLETC has to get its money worth out of the instructors. -He thinks he can get a local volunteer to do it, a high-school student. - -He says a bit more, something I think about an Eagle Scout law-enforcement -liaison program, but my mind has rocketed off in disbelief. - -"You're going to put a TEENAGER in charge of a federal security BBS?" -I'm speechless. It hasn't escaped my notice that the FLETC Financial -Fraud Institute is the ULTIMATE hacker-trashing target; there is stuff in here, -stuff of such utter and consummate cool by every standard of the -digital underground. . . . - -I imagine the hackers of my acquaintance, fainting dead-away from -forbidden-knowledge greed-fits, at the mere prospect of cracking -the superultra top-secret computers used to train the Secret Service -in computer-crime. . . . - -"Uhm, Carlton," I babble, "I'm sure he's a really nice kid and all, -but that's a terrible temptation to set in front of somebody who's, -you know, into computers and just starting out. . . ." - -"Yeah," he says, "that did occur to me." For the first time I begin -to suspect that he's pulling my leg. - -He seems proudest when he shows me an ongoing project called JICC, -Joint Intelligence Control Council. It's based on the services provided -by EPIC, the El Paso Intelligence Center, which supplies data and intelligence -to the Drug Enforcement Administration, the Customs Service, the Coast Guard, -and the state police of the four southern border states. Certain EPIC files -can now be accessed by drug-enforcement police of Central America, -South America and the Caribbean, who can also trade information -among themselves. Using a telecom program called "White Hat," -written by two brothers named Lopez from the Dominican Republic, -police can now network internationally on inexpensive PCs. -Carlton Fitzpatrick is teaching a class of drug-war agents -from the Third World, and he's very proud of their progress. -Perhaps soon the sophisticated smuggling networks of the -Medellin Cartel will be matched by a sophisticated computer -network of the Medellin Cartel's sworn enemies. They'll track boats, -track contraband, track the international drug-lords who now leap over -borders with great ease, defeating the police through the clever use -of fragmented national jurisdictions. - -JICC and EPIC must remain beyond the scope of this book. -They seem to me to be very large topics fraught with complications -that I am not fit to judge. I do know, however, that the international, -computer-assisted networking of police, across national boundaries, -is something that Carlton Fitzpatrick considers very important, -a harbinger of a desirable future. I also know that networks -by their nature ignore physical boundaries. And I also know -that where you put communications you put a community, -and that when those communities become self-aware -they will fight to preserve themselves and to expand their influence. -I make no judgements whether this is good or bad. -It's just cyberspace; it's just the way things are. - -I asked Carlton Fitzpatrick what advice he would have for -a twenty-year-old who wanted to shine someday in the world -of electronic law enforcement. - -He told me that the number one rule was simply not to be -scared of computers. You don't need to be an obsessive -"computer weenie," but you mustn't be buffaloed just because -some machine looks fancy. The advantages computers give -smart crooks are matched by the advantages they give smart cops. -Cops in the future will have to enforce the law "with their heads, -not their holsters." Today you can make good cases without ever -leaving your office. In the future, cops who resist the computer -revolution will never get far beyond walking a beat. - -I asked Carlton Fitzpatrick if he had some single message for the public; -some single thing that he would most like the American public to know -about his work. - -He thought about it while. "Yes," he said finally. "TELL me the rules, -and I'll TEACH those rules!" He looked me straight in the eye. -"I do the best that I can." - - - -PART FOUR: THE CIVIL LIBERTARIANS - - -The story of the Hacker Crackdown, as we have followed it thus far, -has been technological, subcultural, criminal and legal. -The story of the Civil Libertarians, though it partakes -of all those other aspects, is profoundly and thoroughly POLITICAL. - -In 1990, the obscure, long-simmering struggle over the ownership -and nature of cyberspace became loudly and irretrievably public. -People from some of the oddest corners of American society suddenly -found themselves public figures. Some of these people found this -situation much more than they had ever bargained for. They backpedalled, -and tried to retreat back to the mandarin obscurity of their cozy -subcultural niches. This was generally to prove a mistake. - -But the civil libertarians seized the day in 1990. They found themselves -organizing, propagandizing, podium-pounding, persuading, touring, -negotiating, posing for publicity photos, submitting to interviews, -squinting in the limelight as they tried a tentative, but growingly -sophisticated, buck-and-wing upon the public stage. - -It's not hard to see why the civil libertarians should have -this competitive advantage. - -The hackers of the digital underground are an hermetic elite. -They find it hard to make any remotely convincing case for -their actions in front of the general public. Actually, -hackers roundly despise the "ignorant" public, and have never -trusted the judgement of "the system." Hackers do propagandize, -but only among themselves, mostly in giddy, badly spelled manifestos -of class warfare, youth rebellion or naive techie utopianism. -Hackers must strut and boast in order to establish and preserve -their underground reputations. But if they speak out too loudly -and publicly, they will break the fragile surface-tension of the underground, -and they will be harrassed or arrested. Over the longer term, -most hackers stumble, get busted, get betrayed, or simply give up. -As a political force, the digital underground is hamstrung. - -The telcos, for their part, are an ivory tower under protracted seige. -They have plenty of money with which to push their calculated public image, -but they waste much energy and goodwill attacking one another with -slanderous and demeaning ad campaigns. The telcos have suffered -at the hands of politicians, and, like hackers, they don't trust -the public's judgement. And this distrust may be well-founded. -Should the general public of the high-tech 1990s come to understand -its own best interests in telecommunications, that might well pose -a grave threat to the specialized technical power and authority -that the telcos have relished for over a century. The telcos do -have strong advantages: loyal employees, specialized expertise, -influence in the halls of power, tactical allies in law enforcement, -and unbelievably vast amounts of money. But politically speaking, they lack -genuine grassroots support; they simply don't seem to have many friends. - -Cops know a lot of things other people don't know. -But cops willingly reveal only those aspects of their -knowledge that they feel will meet their institutional -purposes and further public order. Cops have respect, -they have responsibilities, they have power in the streets -and even power in the home, but cops don't do particularly -well in limelight. When pressed, they will step out in the -public gaze to threaten bad-guys, or to cajole prominent citizens, -or perhaps to sternly lecture the naive and misguided. -But then they go back within their time-honored fortress -of the station-house, the courtroom and the rule-book. - -The electronic civil libertarians, however, have proven to be -born political animals. They seemed to grasp very early on -the postmodern truism that communication is power. Publicity is power. -Soundbites are power. The ability to shove one's issue onto the public -agenda--and KEEP IT THERE--is power. Fame is power. Simple personal -fluency and eloquence can be power, if you can somehow catch the -public's eye and ear. - -The civil libertarians had no monopoly on "technical power"-- -though they all owned computers, most were not particularly -advanced computer experts. They had a good deal of money, -but nowhere near the earthshaking wealth and the galaxy -of resources possessed by telcos or federal agencies. -They had no ability to arrest people. They carried -out no phreak and hacker covert dirty-tricks. - -But they really knew how to network. - -Unlike the other groups in this book, the civil libertarians -have operated very much in the open, more or less right -in the public hurly-burly. They have lectured audiences galore -and talked to countless journalists, and have learned to -refine their spiels. They've kept the cameras clicking, -kept those faxes humming, swapped that email, -run those photocopiers on overtime, licked envelopes -and spent small fortunes on airfare and long-distance. -In an information society, this open, overt, obvious activity -has proven to be a profound advantage. - -In 1990, the civil libertarians of cyberspace assembled -out of nowhere in particular, at warp speed. This "group" -(actually, a networking gaggle of interested parties -which scarcely deserves even that loose term) has almost nothing -in the way of formal organization. Those formal civil libertarian -organizations which did take an interest in cyberspace issues, -mainly the Computer Professionals for Social Responsibility -and the American Civil Liberties Union, were carried along -by events in 1990, and acted mostly as adjuncts, -underwriters or launching-pads. - -The civil libertarians nevertheless enjoyed the greatest success -of any of the groups in the Crackdown of 1990. At this writing, -their future looks rosy and the political initiative is firmly in their hands. -This should be kept in mind as we study the highly unlikely lives -and lifestyles of the people who actually made this happen. - -# - -In June 1989, Apple Computer, Inc., of Cupertino, -California, had a problem. Someone had illicitly copied -a small piece of Apple's proprietary software, software -which controlled an internal chip driving the Macintosh -screen display. This Color QuickDraw source code was -a closely guarded piece of Apple's intellectual property. -Only trusted Apple insiders were supposed to possess it. - -But the "NuPrometheus League" wanted things otherwise. -This person (or persons) made several illicit copies -of this source code, perhaps as many as two dozen. -He (or she, or they) then put those illicit floppy disks -into envelopes and mailed them to people all over America: -people in the computer industry who were associated with, -but not directly employed by, Apple Computer. - -The NuPrometheus caper was a complex, highly ideological, -and very hacker-like crime. Prometheus, it will be recalled, -stole the fire of the Gods and gave this potent gift to the -general ranks of downtrodden mankind. A similar god-in-the-manger -attitude was implied for the corporate elite of Apple Computer, -while the "Nu" Prometheus had himself cast in the role of rebel demigod. -The illicitly copied data was given away for free. - -The new Prometheus, whoever he was, escaped the -fate of the ancient Greek Prometheus, who was chained -to a rock for centuries by the vengeful gods while an eagle -tore and ate his liver. On the other hand, NuPrometheus -chickened out somewhat by comparison with his role model. -The small chunk of Color QuickDraw code he had filched -and replicated was more or less useless to Apple's -industrial rivals (or, in fact, to anyone else). -Instead of giving fire to mankind, it was more as if -NuPrometheus had photocopied the schematics for part of a Bic lighter. -The act was not a genuine work of industrial espionage. -It was best interpreted as a symbolic, deliberate slap -in the face for the Apple corporate heirarchy. - -Apple's internal struggles were well-known in the industry. Apple's founders, -Jobs and Wozniak, had both taken their leave long since. Their raucous core -of senior employees had been a barnstorming crew of 1960s Californians, -many of them markedly less than happy with the new button-down multimillion -dollar regime at Apple. Many of the programmers and developers who had -invented the Macintosh model in the early 1980s had also taken their leave of -the company. It was they, not the current masters of Apple's corporate fate, -who had invented the stolen Color QuickDraw code. The NuPrometheus stunt -was well-calculated to wound company morale. - -Apple called the FBI. The Bureau takes an interest in high-profile -intellectual-property theft cases, industrial espionage and theft -of trade secrets. These were likely the right people to call, -and rumor has it that the entities responsible were in fact discovered -by the FBI, and then quietly squelched by Apple management. NuPrometheus -was never publicly charged with a crime, or prosecuted, or jailed. -But there were no further illicit releases of Macintosh internal software. -Eventually the painful issue of NuPrometheus was allowed to fade. - -In the meantime, however, a large number of puzzled bystanders -found themselves entertaining surprise guests from the FBI. - -One of these people was John Perry Barlow. Barlow is a most unusual man, -difficult to describe in conventional terms. He is perhaps best known as -a songwriter for the Grateful Dead, for he composed lyrics for -"Hell in a Bucket," "Picasso Moon," "Mexicali Blues," "I Need a Miracle," -and many more; he has been writing for the band since 1970. - -Before we tackle the vexing question as to why a rock lyricist -should be interviewed by the FBI in a computer-crime case, -it might be well to say a word or two about the Grateful Dead. -The Grateful Dead are perhaps the most successful and long-lasting -of the numerous cultural emanations from the Haight-Ashbury district -of San Francisco, in the glory days of Movement politics and -lysergic transcendance. The Grateful Dead are a nexus, a veritable -whirlwind, of applique decals, psychedelic vans, tie-dyed T-shirts, -earth-color denim, frenzied dancing and open and unashamed drug use. -The symbols, and the realities, of Californian freak power surround -the Grateful Dead like knotted macrame. - -The Grateful Dead and their thousands of Deadhead devotees -are radical Bohemians. This much is widely understood. -Exactly what this implies in the 1990s is rather more problematic. - -The Grateful Dead are among the world's most popular -and wealthy entertainers: number 20, according to Forbes magazine, -right between M.C. Hammer and Sean Connery. In 1990, this jeans-clad -group of purported raffish outcasts earned seventeen million dollars. -They have been earning sums much along this line for quite some time now. - -And while the Dead are not investment bankers or three-piece-suit -tax specialists--they are, in point of fact, hippie musicians-- -this money has not been squandered in senseless Bohemian excess. -The Dead have been quietly active for many years, funding various -worthy activities in their extensive and widespread cultural community. - -The Grateful Dead are not conventional players in the American -power establishment. They nevertheless are something of a force -to be reckoned with. They have a lot of money and a lot of friends -in many places, both likely and unlikely. - -The Dead may be known for back-to-the-earth environmentalist rhetoric, -but this hardly makes them anti-technological Luddites. On the contrary, -like most rock musicians, the Grateful Dead have spent their entire adult -lives in the company of complex electronic equipment. They have funds to burn -on any sophisticated tool and toy that might happen to catch their fancy. -And their fancy is quite extensive. - -The Deadhead community boasts any number of recording engineers, -lighting experts, rock video mavens, electronic technicians -of all descriptions. And the drift goes both ways. Steve Wozniak, -Apple's co-founder, used to throw rock festivals. Silicon Valley rocks out. - -These are the 1990s, not the 1960s. Today, for a surprising number of people -all over America, the supposed dividing line between Bohemian and technician -simply no longer exists. People of this sort may have a set of windchimes -and a dog with a knotted kerchief 'round its neck, but they're also quite -likely to own a multimegabyte Macintosh running MIDI synthesizer software -and trippy fractal simulations. These days, even Timothy Leary himself, -prophet of LSD, does virtual-reality computer-graphics demos in -his lecture tours. - -John Perry Barlow is not a member of the Grateful Dead. He is, however, -a ranking Deadhead. - -Barlow describes himself as a "techno-crank." A vague term like -"social activist" might not be far from the mark, either. -But Barlow might be better described as a "poet"--if one keeps in mind -Percy Shelley's archaic definition of poets as "unacknowledged legislators -of the world." - -Barlow once made a stab at acknowledged legislator status. In 1987, -he narrowly missed the Republican nomination for a seat in the -Wyoming State Senate. Barlow is a Wyoming native, the third-generation -scion of a well-to-do cattle-ranching family. He is in his early forties, -married and the father of three daughters. - -Barlow is not much troubled by other people's narrow notions of consistency. -In the late 1980s, this Republican rock lyricist cattle rancher sold his ranch -and became a computer telecommunications devotee. - -The free-spirited Barlow made this transition with ease. He genuinely -enjoyed computers. With a beep of his modem, he leapt from small-town -Pinedale, Wyoming, into electronic contact with a large and lively crowd -of bright, inventive, technological sophisticates from all over the world. -Barlow found the social milieu of computing attractive: its fast-lane pace, -its blue-sky rhetoric, its open-endedness. Barlow began dabbling in -computer journalism, with marked success, as he was a quick study, -and both shrewd and eloquent. He frequently travelled to San Francisco -to network with Deadhead friends. There Barlow made extensive contacts -throughout the Californian computer community, including friendships -among the wilder spirits at Apple. - -In May 1990, Barlow received a visit from a local Wyoming agent of the FBI. -The NuPrometheus case had reached Wyoming. - -Barlow was troubled to find himself under investigation in an -area of his interests once quite free of federal attention. -He had to struggle to explain the very nature of computer-crime -to a headscratching local FBI man who specialized in cattle-rustling. -Barlow, chatting helpfully and demonstrating the wonders of his modem -to the puzzled fed, was alarmed to find all "hackers" generally under -FBI suspicion as an evil influence in the electronic community. -The FBI, in pursuit of a hacker called "NuPrometheus," were tracing -attendees of a suspect group called the Hackers Conference. - -The Hackers Conference, which had been started in 1984, was a -yearly Californian meeting of digital pioneers and enthusiasts. -The hackers of the Hackers Conference had little if anything to do -with the hackers of the digital underground. On the contrary, -the hackers of this conference were mostly well-to-do Californian -high-tech CEOs, consultants, journalists and entrepreneurs. -(This group of hackers were the exact sort of "hackers" -most likely to react with militant fury at any criminal -degradation of the term "hacker.") - -Barlow, though he was not arrested or accused of a crime, -and though his computer had certainly not gone out the door, -was very troubled by this anomaly. He carried the word to the Well. - -Like the Hackers Conference, "the Well" was an emanation of the -Point Foundation. Point Foundation, the inspiration of a wealthy -Californian 60s radical named Stewart Brand, was to be a major -launch-pad of the civil libertarian effort. - -Point Foundation's cultural efforts, like those of their fellow Bay Area -Californians the Grateful Dead, were multifaceted and multitudinous. -Rigid ideological consistency had never been a strong suit of the -Whole Earth Catalog. This Point publication had enjoyed a strong -vogue during the late 60s and early 70s, when it offered hundreds -of practical (and not so practical) tips on communitarian living, -environmentalism, and getting back-to-the-land. The Whole Earth Catalog, -and its sequels, sold two and half million copies and won a -National Book Award. - -With the slow collapse of American radical dissent, the Whole Earth Catalog -had slipped to a more modest corner of the cultural radar; but in its -magazine incarnation, CoEvolution Quarterly, the Point Foundation -continued to offer a magpie potpourri of "access to tools and ideas." - -CoEvolution Quarterly, which started in 1974, was never a widely -popular magazine. Despite periodic outbreaks of millenarian fervor, -CoEvolution Quarterly failed to revolutionize Western civilization -and replace leaden centuries of history with bright new Californian paradigms. -Instead, this propaganda arm of Point Foundation cakewalked a fine line between -impressive brilliance and New Age flakiness. CoEvolution Quarterly carried -no advertising, cost a lot, and came out on cheap newsprint with modest -black-and-white graphics. It was poorly distributed, and spread mostly -by subscription and word of mouth. - -It could not seem to grow beyond 30,000 subscribers. -And yet--it never seemed to shrink much, either. -Year in, year out, decade in, decade out, some strange -demographic minority accreted to support the magazine. -The enthusiastic readership did not seem to have much -in the way of coherent politics or ideals. It was sometimes -hard to understand what held them together (if the often bitter -debate in the letter-columns could be described as "togetherness"). - -But if the magazine did not flourish, it was resilient; it got by. -Then, in 1984, the birth-year of the Macintosh computer, -CoEvolution Quarterly suddenly hit the rapids. Point Foundation -had discovered the computer revolution. Out came the Whole Earth -Software Catalog of 1984, arousing headscratching doubts among -the tie-dyed faithful, and rabid enthusiasm among the nascent -"cyberpunk" milieu, present company included. Point Foundation -started its yearly Hackers Conference, and began to take an -extensive interest in the strange new possibilities of -digital counterculture. CoEvolution Quarterlyfolded its teepee, -replaced by Whole Earth Software Review and eventually by Whole Earth -Review (the magazine's present incarnation, currently under -the editorship of virtual-reality maven Howard Rheingold). - -1985 saw the birth of the "WELL"--the "Whole Earth 'Lectronic Link." -The Well was Point Foundation's bulletin board system. - -As boards went, the Well was an anomaly from the beginning, -and remained one. It was local to San Francisco. -It was huge, with multiple phonelines and enormous files -of commentary. Its complex UNIX-based software might be -most charitably described as "user-opaque." It was run on -a mainframe out of the rambling offices of a non-profit -cultural foundation in Sausalito. And it was crammed with -fans of the Grateful Dead. - -Though the Well was peopled by chattering hipsters of the Bay Area -counterculture, it was by no means a "digital underground" board. -Teenagers were fairly scarce; most Well users (known as "Wellbeings") -were thirty- and forty-something Baby Boomers. They tended to work -in the information industry: hardware, software, telecommunications, -media, entertainment. Librarians, academics, and journalists were -especially common on the Well, attracted by Point Foundation's -open-handed distribution of "tools and ideas." - -There were no anarchy files on the Well, scarcely a -dropped hint about access codes or credit-card theft. -No one used handles. Vicious "flame-wars" were held to -a comparatively civilized rumble. Debates were sometimes sharp, -but no Wellbeing ever claimed that a rival had disconnected his phone, -trashed his house, or posted his credit card numbers. - -The Well grew slowly as the 1980s advanced. It charged a modest sum -for access and storage, and lost money for years--but not enough to hamper -the Point Foundation, which was nonprofit anyway. By 1990, the Well -had about five thousand users. These users wandered about a gigantic -cyberspace smorgasbord of "Conferences", each conference itself consisting -of a welter of "topics," each topic containing dozens, sometimes hundreds -of comments, in a tumbling, multiperson debate that could last for months -or years on end. - - -In 1991, the Well's list of conferences looked like this: - - -CONFERENCES ON THE WELL - -WELL "Screenzine" Digest (g zine) - -Best of the WELL - vintage material - (g best) - -Index listing of new topics in all conferences - (g newtops) - -Business - Education ----------------------- - -Apple Library Users Group(g alug) Agriculture (g agri) -Brainstorming (g brain) Classifieds (g cla) -Computer Journalism (g cj) Consultants (g consult) -Consumers (g cons) Design (g design) -Desktop Publishing (g desk) Disability (g disability) -Education (g ed) Energy (g energy91) -Entrepreneurs (g entre) Homeowners (g home) -Indexing (g indexing) Investments (g invest) -Kids91 (g kids) Legal (g legal) -One Person Business (g one) -Periodical/newsletter (g per) -Telecomm Law (g tcl) The Future (g fut) -Translators (g trans) Travel (g tra) -Work (g work) - -Electronic Frontier Foundation (g eff) -Computers, Freedom & Privacy (g cfp) -Computer Professionals for Social Responsibility (g cpsr) - -Social - Political - Humanities ---------------------------------- - -Aging (g gray) AIDS (g aids) -Amnesty International (g amnesty) Archives (g arc) -Berkeley (g berk) Buddhist (g wonderland) -Christian (g cross) Couples (g couples) -Current Events (g curr) Dreams (g dream) -Drugs (g dru) East Coast (g east) -Emotional Health@@@@ (g private) Erotica (g eros) -Environment (g env) Firearms (g firearms) -First Amendment (g first) Fringes of Reason (g fringes) -Gay (g gay) Gay (Private)# (g gaypriv) -Geography (g geo) German (g german) -Gulf War (g gulf) Hawaii (g aloha) -Health (g heal) History (g hist) -Holistic (g holi) Interview (g inter) -Italian (g ital) Jewish (g jew) -Liberty (g liberty) Mind (g mind) -Miscellaneous (g misc) Men on the WELL@@ (g mow) -Network Integration (g origin) Nonprofits (g non) -North Bay (g north) Northwest (g nw) -Pacific Rim (g pacrim) Parenting (g par) -Peace (g pea) Peninsula (g pen) -Poetry (g poetry) Philosophy (g phi) -Politics (g pol) Psychology (g psy) -Psychotherapy (g therapy) Recovery## (g recovery) -San Francisco (g sanfran) Scams (g scam) -Sexuality (g sex) Singles (g singles) -Southern (g south) Spanish (g spanish) -Spirituality (g spirit) Tibet (g tibet) -Transportation (g transport) True Confessions (g tru) -Unclear (g unclear) WELL Writer's Workshop@@@(g www) -Whole Earth (g we) Women on the WELL@(g wow) -Words (g words) Writers (g wri) - -@@@@Private Conference - mail wooly for entry -@@@Private conference - mail sonia for entry -@@Private conference - mail flash for entry -@ Private conference - mail reva for entry -# Private Conference - mail hudu for entry -## Private Conference - mail dhawk for entry - -Arts - Recreation - Entertainment ------------------------------------ -ArtCom Electronic Net (g acen) -Audio-Videophilia (g aud) -Bicycles (g bike) Bay Area Tonight@@(g bat) -Boating (g wet) Books (g books) -CD's (g cd) Comics (g comics) -Cooking (g cook) Flying (g flying) -Fun (g fun) Games (g games) -Gardening (g gard) Kids (g kids) -Nightowls@ (g owl) Jokes (g jokes) -MIDI (g midi) Movies (g movies) -Motorcycling (g ride) Motoring (g car) -Music (g mus) On Stage (g onstage) -Pets (g pets) Radio (g rad) -Restaurant (g rest) Science Fiction (g sf) -Sports (g spo) Star Trek (g trek) -Television (g tv) Theater (g theater) -Weird (g weird) Zines/Factsheet Five(g f5) -@Open from midnight to 6am -@@Updated daily - -Grateful Dead -------------- -Grateful Dead (g gd) Deadplan@ (g dp) -Deadlit (g deadlit) Feedback (g feedback) -GD Hour (g gdh) Tapes (g tapes) -Tickets (g tix) Tours (g tours) - -@Private conference - mail tnf for entry - -Computers ------------ -AI/Forth/Realtime (g realtime) Amiga (g amiga) -Apple (g app) Computer Books (g cbook) -Art & Graphics (g gra) Hacking (g hack) -HyperCard (g hype) IBM PC (g ibm) -LANs (g lan) Laptop (g lap) -Macintosh (g mac) Mactech (g mactech) -Microtimes (g microx) Muchomedia (g mucho) -NeXt (g next) OS/2 (g os2) -Printers (g print) Programmer's Net (g net) -Siggraph (g siggraph) Software Design (g sdc) -Software/Programming (g software) -Software Support (g ssc) -Unix (g unix) Windows (g windows) -Word Processing (g word) - -Technical - Communications ----------------------------- -Bioinfo (g bioinfo) Info (g boing) -Media (g media) NAPLPS (g naplps) -Netweaver (g netweaver) Networld (g networld) -Packet Radio (g packet) Photography (g pho) -Radio (g rad) Science (g science) -Technical Writers (g tec) Telecommunications(g tele) -Usenet (g usenet) Video (g vid) -Virtual Reality (g vr) - -The WELL Itself ---------------- -Deeper (g deeper) Entry (g ent) -General (g gentech) Help (g help) -Hosts (g hosts) Policy (g policy) -System News (g news) Test (g test) - -The list itself is dazzling, bringing to the untutored eye -a dizzying impression of a bizarre milieu of mountain-climbing -Hawaiian holistic photographers trading true-life confessions -with bisexual word-processing Tibetans. - -But this confusion is more apparent than real. Each of these conferences -was a little cyberspace world in itself, comprising dozens and perhaps -hundreds of sub-topics. Each conference was commonly frequented by -a fairly small, fairly like-minded community of perhaps a few dozen people. -It was humanly impossible to encompass the entire Well (especially since -access to the Well's mainframe computer was billed by the hour). -Most long-time users contented themselves with a few favorite -topical neighborhoods, with the occasional foray elsewhere -for a taste of exotica. But especially important news items, -and hot topical debates, could catch the attention of the entire -Well community. - -Like any community, the Well had its celebrities, and John Perry Barlow, -the silver-tongued and silver-modemed lyricist of the Grateful Dead, -ranked prominently among them. It was here on the Well that Barlow -posted his true-life tale of computer-crime encounter with the FBI. - -The story, as might be expected, created a great stir. The Well was -already primed for hacker controversy. In December 1989, Harper's magazine -had hosted a debate on the Well about the ethics of illicit computer intrusion. -While over forty various computer-mavens took part, Barlow proved a star -in the debate. So did "Acid Phreak" and "Phiber Optik," a pair of young -New York hacker-phreaks whose skills at telco switching-station intrusion -were matched only by their apparently limitless hunger for fame. -The advent of these two boldly swaggering outlaws in the precincts -of the Well created a sensation akin to that of Black Panthers -at a cocktail party for the radically chic. - -Phiber Optik in particular was to seize the day in 1990. -A devotee of the 2600 circle and stalwart of the New York -hackers' group "Masters of Deception," Phiber Optik was -a splendid exemplar of the computer intruder as committed dissident. -The eighteen-year-old Optik, a high-school dropout and part-time -computer repairman, was young, smart, and ruthlessly obsessive, -a sharp-dressing, sharp-talking digital dude who was utterly -and airily contemptuous of anyone's rules but his own. -By late 1991, Phiber Optik had appeared in Harper's, -Esquire, The New York Times, in countless public debates -and conventions, even on a television show hosted by Geraldo Rivera. - -Treated with gingerly respect by Barlow and other Well mavens, -Phiber Optik swiftly became a Well celebrity. Strangely, despite -his thorny attitude and utter single-mindedness, Phiber Optik seemed -to arouse strong protective instincts in most of the people who met him. -He was great copy for journalists, always fearlessly ready to swagger, -and, better yet, to actually DEMONSTRATE some off-the-wall digital stunt. -He was a born media darling. - -Even cops seemed to recognize that there was something peculiarly unworldly -and uncriminal about this particular troublemaker. He was so bold, -so flagrant, so young, and so obviously doomed, that even those -who strongly disapproved of his actions grew anxious for his welfare, -and began to flutter about him as if he were an endangered seal pup. - -In January 24, 1990 (nine days after the Martin Luther King Day Crash), -Phiber Optik, Acid Phreak, and a third NYC scofflaw named Scorpion were -raided by the Secret Service. Their computers went out the door, -along with the usual blizzard of papers, notebooks, compact disks, -answering machines, Sony Walkmans, etc. Both Acid Phreak and -Phiber Optik were accused of having caused the Crash. - -The mills of justice ground slowly. The case eventually fell into -the hands of the New York State Police. Phiber had lost his machinery -in the raid, but there were no charges filed against him for over a year. -His predicament was extensively publicized on the Well, where it caused -much resentment for police tactics. It's one thing to merely hear about -a hacker raided or busted; it's another to see the police attacking someone -you've come to know personally, and who has explained his motives at length. -Through the Harper's debate on the Well, it had become clear to the -Wellbeings that Phiber Optik was not in fact going to "hurt anything." -In their own salad days, many Wellbeings had tasted tear-gas in pitched -street-battles with police. They were inclined to indulgence for -acts of civil disobedience. - -Wellbeings were also startled to learn of the draconian thoroughness -of a typical hacker search-and-seizure. It took no great stretch of -imagination for them to envision themselves suffering much the same treatment. - -As early as January 1990, sentiment on the Well had already begun to sour, -and people had begun to grumble that "hackers" were getting a raw deal -from the ham-handed powers-that-be. The resultant issue of Harper's -magazine posed the question as to whether computer-intrusion was a "crime" -at all. As Barlow put it later: "I've begun to wonder if we wouldn't -also regard spelunkers as desperate criminals if AT&T owned all the caves." - -In February 1991, more than a year after the raid on his home, -Phiber Optik was finally arrested, and was charged with first-degree -Computer Tampering and Computer Trespass, New York state offenses. -He was also charged with a theft-of-service misdemeanor, involving a complex -free-call scam to a 900 number. Phiber Optik pled guilty to the misdemeanor -charge, and was sentenced to 35 hours of community service. - -This passing harassment from the unfathomable world of straight people -seemed to bother Optik himself little if at all. Deprived of his computer -by the January search-and-seizure, he simply bought himself a portable -computer so the cops could no longer monitor the phone where he lived -with his Mom, and he went right on with his depredations, sometimes on -live radio or in front of television cameras. - -The crackdown raid may have done little to dissuade Phiber Optik, -but its galling affect on the Wellbeings was profound. As 1990 rolled on, -the slings and arrows mounted: the Knight Lightning raid, -the Steve Jackson raid, the nation-spanning Operation Sundevil. -The rhetoric of law enforcement made it clear that there was, -in fact, a concerted crackdown on hackers in progress. - -The hackers of the Hackers Conference, the Wellbeings, and their ilk, -did not really mind the occasional public misapprehension of "hacking;" -if anything, this membrane of differentiation from straight society -made the "computer community" feel different, smarter, better. -They had never before been confronted, however, by a concerted -vilification campaign. - -Barlow's central role in the counter-struggle was one of the major -anomalies of 1990. Journalists investigating the controversy -often stumbled over the truth about Barlow, but they commonly -dusted themselves off and hurried on as if nothing had happened. -It was as if it were TOO MUCH TO BELIEVE that a 1960s freak -from the Grateful Dead had taken on a federal law enforcement operation -head-to-head and ACTUALLY SEEMED TO BE WINNING! - -Barlow had no easily detectable power-base for a political struggle -of this kind. He had no formal legal or technical credentials. -Barlow was, however, a computer networker of truly stellar brilliance. -He had a poet's gift of concise, colorful phrasing. He also had a -journalist's shrewdness, an off-the-wall, self-deprecating wit, -and a phenomenal wealth of simple personal charm. - -The kind of influence Barlow possessed is fairly common currency -in literary, artistic, or musical circles. A gifted critic can -wield great artistic influence simply through defining -the temper of the times, by coining the catch-phrases -and the terms of debate that become the common currency of the period. -(And as it happened, Barlow WAS a part-time art critic, -with a special fondness for the Western art of Frederic Remington.) - -Barlow was the first commentator to adopt William Gibson's -striking science-fictional term "cyberspace" as a synonym -for the present-day nexus of computer and telecommunications networks. -Barlow was insistent that cyberspace should be regarded as -a qualitatively new world, a "frontier." According to Barlow, -the world of electronic communications, now made visible through -the computer screen, could no longer be usefully regarded -as just a tangle of high-tech wiring. Instead, it had become -a PLACE, cyberspace, which demanded a new set of metaphors, -a new set of rules and behaviors. The term, as Barlow employed it, -struck a useful chord, and this concept of cyberspace was picked up -by Time, Scientific American, computer police, hackers, and even -Constitutional scholars. "Cyberspace" now seems likely to become -a permanent fixture of the language. - -Barlow was very striking in person: a tall, craggy-faced, bearded, -deep-voiced Wyomingan in a dashing Western ensemble of jeans, jacket, -cowboy boots, a knotted throat-kerchief and an ever-present Grateful Dead -cloisonne lapel pin. - -Armed with a modem, however, Barlow was truly in his element. -Formal hierarchies were not Barlow's strong suit; he rarely missed -a chance to belittle the "large organizations and their drones," -with their uptight, institutional mindset. Barlow was very much -of the free-spirit persuasion, deeply unimpressed by brass-hats -and jacks-in-office. But when it came to the digital grapevine, -Barlow was a cyberspace ad-hocrat par excellence. - -There was not a mighty army of Barlows. There was only one Barlow, -and he was a fairly anomolous individual. However, the situation only -seemed to REQUIRE a single Barlow. In fact, after 1990, many people -must have concluded that a single Barlow was far more than -they'd ever bargained for. - -Barlow's querulous mini-essay about his encounter with the FBI -struck a strong chord on the Well. A number of other free spirits -on the fringes of Apple Computing had come under suspicion, -and they liked it not one whit better than he did. - -One of these was Mitchell Kapor, the co-inventor of the spreadsheet -program "Lotus 1-2-3" and the founder of Lotus Development Corporation. -Kapor had written-off the passing indignity of being fingerprinted -down at his own local Boston FBI headquarters, but Barlow's post -made the full national scope of the FBI's dragnet clear to Kapor. -The issue now had Kapor's full attention. As the Secret Service -swung into anti-hacker operation nationwide in 1990, Kapor watched -every move with deep skepticism and growing alarm. - -As it happened, Kapor had already met Barlow, who had interviewed Kapor -for a California computer journal. Like most people who met Barlow, -Kapor had been very taken with him. Now Kapor took it upon himself -to drop in on Barlow for a heart-to-heart talk about the situation. - -Kapor was a regular on the Well. Kapor had been a devotee of the -Whole Earth Catalogsince the beginning, and treasured a complete run -of the magazine. And Kapor not only had a modem, but a private jet. -In pursuit of the scattered high-tech investments of Kapor Enterprises Inc., -his personal, multi-million dollar holding company, Kapor commonly crossed -state lines with about as much thought as one might give to faxing a letter. - -The Kapor-Barlow council of June 1990, in Pinedale, Wyoming, was the start -of the Electronic Frontier Foundation. Barlow swiftly wrote a manifesto, -"Crime and Puzzlement," which announced his, and Kapor's, intention -to form a political organization to "raise and disburse funds for education, -lobbying, and litigation in the areas relating to digital speech and the -extension of the Constitution into Cyberspace." - -Furthermore, proclaimed the manifesto, the foundation would -"fund, conduct, and support legal efforts to demonstrate -that the Secret Service has exercised prior restraint on publications, -limited free speech, conducted improper seizure of equipment and data, -used undue force, and generally conducted itself in a fashion which -is arbitrary, oppressive, and unconstitutional." - -"Crime and Puzzlement" was distributed far and wide through computer -networking channels, and also printed in the Whole Earth Review. -The sudden declaration of a coherent, politicized counter-strike -from the ranks of hackerdom electrified the community. Steve Wozniak -(perhaps a bit stung by the NuPrometheus scandal) swiftly offered -to match any funds Kapor offered the Foundation. - -John Gilmore, one of the pioneers of Sun Microsystems, immediately offered -his own extensive financial and personal support. Gilmore, an ardent -libertarian, was to prove an eloquent advocate of electronic privacy issues, -especially freedom from governmental and corporate computer-assisted -surveillance of private citizens. - -A second meeting in San Francisco rounded up further allies: -Stewart Brand of the Point Foundation, virtual-reality pioneers -Jaron Lanier and Chuck Blanchard, network entrepreneur and venture -capitalist Nat Goldhaber. At this dinner meeting, the activists settled on -a formal title: the Electronic Frontier Foundation, Incorporated. -Kapor became its president. A new EFF Conference was opened on -the Point Foundation's Well, and the Well was declared -"the home of the Electronic Frontier Foundation." - -Press coverage was immediate and intense. Like their -nineteenth-century spiritual ancestors, Alexander Graham Bell -and Thomas Watson, the high-tech computer entrepreneurs -of the 1970s and 1980s--people such as Wozniak, Jobs, Kapor, -Gates, and H. Ross Perot, who had raised themselves by their bootstraps -to dominate a glittering new industry--had always made very good copy. - -But while the Wellbeings rejoiced, the press in general seemed -nonplussed by the self-declared "civilizers of cyberspace." -EFF's insistence that the war against "hackers" involved grave -Constitutional civil liberties issues seemed somewhat farfetched, -especially since none of EFF's organizers were lawyers -or established politicians. The business press in particular -found it easier to seize on the apparent core of the story-- -that high-tech entrepreneur Mitchell Kapor had established -a "defense fund for hackers." Was EFF a genuinely important -political development--or merely a clique of wealthy eccentrics, -dabbling in matters better left to the proper authorities? -The jury was still out. - -But the stage was now set for open confrontation. -And the first and the most critical battle was the -hacker show-trial of "Knight Lightning." - -# - -It has been my practice throughout this book to refer to hackers -only by their "handles." There is little to gain by giving -the real names of these people, many of whom are juveniles, -many of whom have never been convicted of any crime, and many -of whom had unsuspecting parents who have already suffered enough. - -But the trial of Knight Lightning on July 24-27, 1990, -made this particular "hacker" a nationally known public figure. -It can do no particular harm to himself or his family if I repeat -the long-established fact that his name is Craig Neidorf (pronounced NYE-dorf). - -Neidorf's jury trial took place in the United States District Court, -Northern District of Illinois, Eastern Division, with the -Honorable Nicholas J. Bua presiding. The United States of America -was the plaintiff, the defendant Mr. Neidorf. The defendant's attorney -was Sheldon T. Zenner of the Chicago firm of Katten, Muchin and Zavis. - -The prosecution was led by the stalwarts of the Chicago Computer Fraud -and Abuse Task Force: William J. Cook, Colleen D. Coughlin, and -David A. Glockner, all Assistant United States Attorneys. -The Secret Service Case Agent was Timothy M. Foley. - -It will be recalled that Neidorf was the co-editor of an underground hacker -"magazine" called Phrack. Phrack was an entirely electronic publication, -distributed through bulletin boards and over electronic networks. -It was amateur publication given away for free. Neidorf had never made -any money for his work in Phrack. Neither had his unindicted co-editor -"Taran King" or any of the numerous Phrack contributors. - -The Chicago Computer Fraud and Abuse Task Force, however, -had decided to prosecute Neidorf as a fraudster. -To formally admit that Phrack was a "magazine" -and Neidorf a "publisher" was to open a prosecutorial -Pandora's Box of First Amendment issues. To do this -was to play into the hands of Zenner and his EFF advisers, -which now included a phalanx of prominent New York civil rights -lawyers as well as the formidable legal staff of Katten, Muchin and Zavis. -Instead, the prosecution relied heavily on the issue of access device fraud: -Section 1029 of Title 18, the section from which the Secret Service drew -its most direct jurisdiction over computer crime. - -Neidorf's alleged crimes centered around the E911 Document. -He was accused of having entered into a fraudulent scheme with the Prophet, -who, it will be recalled, was the Atlanta LoD member who had illicitly -copied the E911 Document from the BellSouth AIMSX system. - -The Prophet himself was also a co-defendant in the Neidorf case, -part-and-parcel of the alleged "fraud scheme" to "steal" BellSouth's -E911 Document (and to pass the Document across state lines, -which helped establish the Neidorf trial as a federal case). -The Prophet, in the spirit of full co-operation, had agreed -to testify against Neidorf. - -In fact, all three of the Atlanta crew stood ready to testify against Neidorf. -Their own federal prosecutors in Atlanta had charged the Atlanta Three with: -(a) conspiracy, (b) computer fraud, (c) wire fraud, (d) access device fraud, -and (e) interstate transportation of stolen property (Title 18, Sections 371, -1030, 1343, 1029, and 2314). - -Faced with this blizzard of trouble, Prophet and Leftist had ducked -any public trial and had pled guilty to reduced charges--one conspiracy -count apiece. Urvile had pled guilty to that odd bit of Section 1029 -which makes it illegal to possess "fifteen or more" illegal access devices -(in his case, computer passwords). And their sentences were scheduled -for September 14, 1990--well after the Neidorf trial. As witnesses, -they could presumably be relied upon to behave. - -Neidorf, however, was pleading innocent. Most everyone else caught up -in the crackdown had "cooperated fully" and pled guilty in hope -of reduced sentences. (Steve Jackson was a notable exception, -of course, and had strongly protested his innocence from the -very beginning. But Steve Jackson could not get a day in court-- -Steve Jackson had never been charged with any crime in the first place.) - -Neidorf had been urged to plead guilty. But Neidorf was a political science -major and was disinclined to go to jail for "fraud" when he had not made -any money, had not broken into any computer, and had been publishing -a magazine that he considered protected under the First Amendment. - -Neidorf's trial was the ONLY legal action of the entire Crackdown -that actually involved bringing the issues at hand out for a public test -in front of a jury of American citizens. - -Neidorf, too, had cooperated with investigators. He had voluntarily -handed over much of the evidence that had led to his own indictment. -He had already admitted in writing that he knew that the E911 Document -had been stolen before he had "published" it in Phrack--or, from the -prosecution's point of view, illegally transported stolen property by wire -in something purporting to be a "publication." - -But even if the "publication" of the E911 Document was not held to be a crime, -that wouldn't let Neidorf off the hook. Neidorf had still received -the E911 Document when Prophet had transferred it to him from Rich Andrews' -Jolnet node. On that occasion, it certainly hadn't been "published"-- -it was hacker booty, pure and simple, transported across state lines. - -The Chicago Task Force led a Chicago grand jury to indict Neidorf -on a set of charges that could have put him in jail for thirty years. -When some of these charges were successfully challenged before Neidorf -actually went to trial, the Chicago Task Force rearranged his -indictment so that he faced a possible jail term of over sixty years! -As a first offender, it was very unlikely that Neidorf would in fact -receive a sentence so drastic; but the Chicago Task Force clearly -intended to see Neidorf put in prison, and his conspiratorial "magazine" -put permanently out of commission. This was a federal case, and Neidorf -was charged with the fraudulent theft of property worth almost -eighty thousand dollars. - -William Cook was a strong believer in high-profile prosecutions -with symbolic overtones. He often published articles on his work -in the security trade press, arguing that "a clear message had -to be sent to the public at large and the computer community -in particular that unauthorized attacks on computers and the theft -of computerized information would not be tolerated by the courts." - -The issues were complex, the prosecution's tactics somewhat unorthodox, -but the Chicago Task Force had proved sure-footed to date. "Shadowhawk" -had been bagged on the wing in 1989 by the Task Force, and sentenced -to nine months in prison, and a $10,000 fine. The Shadowhawk case involved -charges under Section 1030, the "federal interest computer" section. - -Shadowhawk had not in fact been a devotee of "federal-interest" computers -per se. On the contrary, Shadowhawk, who owned an AT&T home computer, -seemed to cherish a special aggression toward AT&T. He had bragged on -the underground boards "Phreak Klass 2600" and "Dr. Ripco" of his skills -at raiding AT&T, and of his intention to crash AT&T's national phone system. -Shadowhawk's brags were noticed by Henry Kluepfel of Bellcore Security, -scourge of the outlaw boards, whose relations with the Chicago Task Force -were long and intimate. - -The Task Force successfully established that Section 1030 applied to -the teenage Shadowhawk, despite the objections of his defense attorney. -Shadowhawk had entered a computer "owned" by U.S. Missile Command -and merely "managed" by AT&T. He had also entered an AT&T computer -located at Robbins Air Force Base in Georgia. Attacking AT&T was -of "federal interest" whether Shadowhawk had intended it or not. - -The Task Force also convinced the court that a piece of AT&T -software that Shadowhawk had illicitly copied from Bell Labs, -the "Artificial Intelligence C5 Expert System," was worth a cool -one million dollars. Shadowhawk's attorney had argued that -Shadowhawk had not sold the program and had made no profit from -the illicit copying. And in point of fact, the C5 Expert System -was experimental software, and had no established market value -because it had never been on the market in the first place. -AT&T's own assessment of a "one million dollar" figure for its -own intangible property was accepted without challenge -by the court, however. And the court concurred with -the government prosecutors that Shadowhawk showed clear -"intent to defraud" whether he'd gotten any money or not. -Shadowhawk went to jail. - -The Task Force's other best-known triumph had been the conviction -and jailing of "Kyrie." Kyrie, a true denizen of the digital -criminal underground, was a 36-year-old Canadian woman, -convicted and jailed for telecommunications fraud in Canada. -After her release from prison, she had fled the wrath of Canada Bell -and the Royal Canadian Mounted Police, and eventually settled, -very unwisely, in Chicago. - -"Kyrie," who also called herself "Long Distance Information," -specialized in voice-mail abuse. She assembled large numbers -of hot long-distance codes, then read them aloud into a series -of corporate voice-mail systems. Kyrie and her friends were -electronic squatters in corporate voice-mail systems, -using them much as if they were pirate bulletin boards, -then moving on when their vocal chatter clogged the system -and the owners necessarily wised up. Kyrie's camp followers -were a loose tribe of some hundred and fifty phone-phreaks, -who followed her trail of piracy from machine to machine, -ardently begging for her services and expertise. - -Kyrie's disciples passed her stolen credit-card numbers, -in exchange for her stolen "long distance information." -Some of Kyrie's clients paid her off in cash, by scamming -credit-card cash advances from Western Union. - -Kyrie travelled incessantly, mostly through airline tickets -and hotel rooms that she scammed through stolen credit cards. -Tiring of this, she found refuge with a fellow female phone -phreak in Chicago. Kyrie's hostess, like a surprising number -of phone phreaks, was blind. She was also physically disabled. -Kyrie allegedly made the best of her new situation by applying for, -and receiving, state welfare funds under a false identity as -a qualified caretaker for the handicapped. - -Sadly, Kyrie's two children by a former marriage had also vanished -underground with her; these pre-teen digital refugees had no legal -American identity, and had never spent a day in school. - -Kyrie was addicted to technical mastery and enthralled by her own -cleverness and the ardent worship of her teenage followers. -This foolishly led her to phone up Gail Thackeray in Arizona, -to boast, brag, strut, and offer to play informant. -Thackeray, however, had already learned far more -than enough about Kyrie, whom she roundly despised -as an adult criminal corrupting minors, a "female Fagin." -Thackeray passed her tapes of Kyrie's boasts to the Secret Service. - -Kyrie was raided and arrested in Chicago in May 1989. -She confessed at great length and pled guilty. - -In August 1990, Cook and his Task Force colleague Colleen Coughlin -sent Kyrie to jail for 27 months, for computer and telecommunications fraud. -This was a markedly severe sentence by the usual wrist-slapping standards -of "hacker" busts. Seven of Kyrie's foremost teenage disciples were also -indicted and convicted. The Kyrie "high-tech street gang," as Cook -described it, had been crushed. Cook and his colleagues had been -the first ever to put someone in prison for voice-mail abuse. -Their pioneering efforts had won them attention and kudos. - -In his article on Kyrie, Cook drove the message home to the readers -of Security Management magazine, a trade journal for corporate -security professionals. The case, Cook said, and Kyrie's stiff sentence, -"reflect a new reality for hackers and computer crime victims in the -'90s. . . . Individuals and corporations who report computer -and telecommunications crimes can now expect that their cooperation -with federal law enforcement will result in meaningful punishment. -Companies and the public at large must report computer-enhanced -crimes if they want prosecutors and the course to protect their rights -to the tangible and intangible property developed and stored on computers." - -Cook had made it his business to construct this "new reality for hackers." -He'd also made it his business to police corporate property rights -to the intangible. - -Had the Electronic Frontier Foundation been a "hacker defense fund" -as that term was generally understood, they presumably would have stood up -for Kyrie. Her 1990 sentence did indeed send a "message" that federal heat -was coming down on "hackers." But Kyrie found no defenders at EFF, -or anywhere else, for that matter. EFF was not a bail-out fund -for electronic crooks. - -The Neidorf case paralleled the Shadowhawk case in certain ways. -The victim once again was allowed to set the value of the "stolen" property. -Once again Kluepfel was both investigator and technical advisor. -Once again no money had changed hands, but the "intent to defraud" was central. - -The prosecution's case showed signs of weakness early on. The Task Force -had originally hoped to prove Neidorf the center of a nationwide -Legion of Doom criminal conspiracy. The Phrack editors threw physical -get-togethers every summer, which attracted hackers from across the country; -generally two dozen or so of the magazine's favorite contributors and readers. -(Such conventions were common in the hacker community; 2600 Magazine, -for instance, held public meetings of hackers in New York, every month.) -LoD heavy-dudes were always a strong presence at these Phrack-sponsored -"Summercons." - -In July 1988, an Arizona hacker named "Dictator" attended Summercon -in Neidorf's home town of St. Louis. Dictator was one of Gail Thackeray's -underground informants; Dictator's underground board in Phoenix was -a sting operation for the Secret Service. Dictator brought an undercover -crew of Secret Service agents to Summercon. The agents bored spyholes -through the wall of Dictator's hotel room in St Louis, and videotaped -the frolicking hackers through a one-way mirror. As it happened, -however, nothing illegal had occurred on videotape, other than the -guzzling of beer by a couple of minors. Summercons were social events, -not sinister cabals. The tapes showed fifteen hours of raucous laughter, -pizza-gobbling, in-jokes and back-slapping. - -Neidorf's lawyer, Sheldon Zenner, saw the Secret Service tapes -before the trial. Zenner was shocked by the complete harmlessness -of this meeting, which Cook had earlier characterized as a sinister -interstate conspiracy to commit fraud. Zenner wanted to show the -Summercon tapes to the jury. It took protracted maneuverings -by the Task Force to keep the tapes from the jury as "irrelevant." - -The E911 Document was also proving a weak reed. It had originally -been valued at $79,449. Unlike Shadowhawk's arcane Artificial Intelligence -booty, the E911 Document was not software--it was written in English. -Computer-knowledgeable people found this value--for a twelve-page -bureaucratic document--frankly incredible. In his "Crime and Puzzlement" -manifesto for EFF, Barlow commented: "We will probably never know how -this figure was reached or by whom, though I like to imagine an appraisal -team consisting of Franz Kafka, Joseph Heller, and Thomas Pynchon." - -As it happened, Barlow was unduly pessimistic. The EFF did, in fact, -eventually discover exactly how this figure was reached, and by whom-- -but only in 1991, long after the Neidorf trial was over. - -Kim Megahee, a Southern Bell security manager, -had arrived at the document's value by simply adding up -the "costs associated with the production" of the E911 Document. -Those "costs" were as follows: - -1. A technical writer had been hired to research and write the E911 Document. - 200 hours of work, at $35 an hour, cost : $7,000. A Project Manager had - overseen the technical writer. 200 hours, at $31 an hour, made: $6,200. - -2. A week of typing had cost $721 dollars. A week of formatting had - cost $721. A week of graphics formatting had cost $742. - -3. Two days of editing cost $367. - -4. A box of order labels cost five dollars. - -5. Preparing a purchase order for the Document, including typing - and the obtaining of an authorizing signature from within the - BellSouth bureaucracy, cost $129. - -6. Printing cost $313. Mailing the Document to fifty people - took fifty hours by a clerk, and cost $858. - -7. Placing the Document in an index took two clerks an hour each, - totalling $43. - -Bureaucratic overhead alone, therefore, was alleged to have cost -a whopping $17,099. According to Mr. Megahee, the typing -of a twelve-page document had taken a full week. Writing it -had taken five weeks, including an overseer who apparently -did nothing else but watch the author for five weeks. -Editing twelve pages had taken two days. Printing and mailing -an electronic document (which was already available on the -Southern Bell Data Network to any telco employee who needed it), -had cost over a thousand dollars. - -But this was just the beginning. There were also the HARDWARE EXPENSES. -Eight hundred fifty dollars for a VT220 computer monitor. -THIRTY-ONE THOUSAND DOLLARS for a sophisticated VAXstation II computer. -Six thousand dollars for a computer printer. TWENTY-TWO THOUSAND DOLLARS -for a copy of "Interleaf" software. Two thousand five hundred dollars -for VMS software. All this to create the twelve-page Document. - -Plus ten percent of the cost of the software and the hardware, for maintenance. -(Actually, the ten percent maintenance costs, though mentioned, had been left -off the final $79,449 total, apparently through a merciful oversight). - -Mr. Megahee's letter had been mailed directly to William Cook himself, -at the office of the Chicago federal attorneys. The United States Government -accepted these telco figures without question. - -As incredulity mounted, the value of the E911 Document was officially -revised downward. This time, Robert Kibler of BellSouth Security -estimated the value of the twelve pages as a mere $24,639.05--based, -purportedly, on "R&D costs." But this specific estimate, -right down to the nickel, did not move the skeptics at all; -in fact it provoked open scorn and a torrent of sarcasm. - -The financial issues concerning theft of proprietary information -have always been peculiar. It could be argued that BellSouth -had not "lost" its E911 Document at all in the first place, -and therefore had not suffered any monetary damage from this "theft." -And Sheldon Zenner did in fact argue this at Neidorf's trial-- -that Prophet's raid had not been "theft," but was better understood -as illicit copying. - -The money, however, was not central to anyone's true purposes in this trial. -It was not Cook's strategy to convince the jury that the E911 Document -was a major act of theft and should be punished for that reason alone. -His strategy was to argue that the E911 Document was DANGEROUS. -It was his intention to establish that the E911 Document was "a road-map" -to the Enhanced 911 System. Neidorf had deliberately and recklessly -distributed a dangerous weapon. Neidorf and the Prophet did not care -(or perhaps even gloated at the sinister idea) that the E911 Document -could be used by hackers to disrupt 911 service, "a life line for every -person certainly in the Southern Bell region of the United States, -and indeed, in many communities throughout the United States," -in Cook's own words. Neidorf had put people's lives in danger. - -In pre-trial maneuverings, Cook had established that the E911 Document -was too hot to appear in the public proceedings of the Neidorf trial. -The JURY ITSELF would not be allowed to ever see this Document, -lest it slip into the official court records, and thus into the hands -of the general public, and, thus, somehow, to malicious hackers -who might lethally abuse it. - -Hiding the E911 Document from the jury may have been a -clever legal maneuver, but it had a severe flaw. There were, -in point of fact, hundreds, perhaps thousands, of people, -already in possession of the E911 Document, just as Phrack -had published it. Its true nature was already obvious -to a wide section of the interested public (all of whom, -by the way, were, at least theoretically, party to -a gigantic wire-fraud conspiracy). Most everyone -in the electronic community who had a modem and any -interest in the Neidorf case already had a copy of the Document. -It had already been available in Phrack for over a year. - -People, even quite normal people without any particular -prurient interest in forbidden knowledge, did not shut their eyes -in terror at the thought of beholding a "dangerous" document -from a telephone company. On the contrary, they tended to trust -their own judgement and simply read the Document for themselves. -And they were not impressed. - -One such person was John Nagle. Nagle was a forty-one-year-old -professional programmer with a masters' degree in computer science -from Stanford. He had worked for Ford Aerospace, where he had invented -a computer-networking technique known as the "Nagle Algorithm," -and for the prominent Californian computer-graphics firm "Autodesk," -where he was a major stockholder. - -Nagle was also a prominent figure on the Well, much respected -for his technical knowledgeability. - -Nagle had followed the civil-liberties debate closely, -for he was an ardent telecommunicator. He was no particular friend -of computer intruders, but he believed electronic publishing -had a great deal to offer society at large, and attempts -to restrain its growth, or to censor free electronic expression, -strongly roused his ire. - -The Neidorf case, and the E911 Document, were both being discussed -in detail on the Internet, in an electronic publication called Telecom Digest. -Nagle, a longtime Internet maven, was a regular reader of Telecom Digest. -Nagle had never seen a copy of Phrack, but the implications of the case -disturbed him. - -While in a Stanford bookstore hunting books on robotics, -Nagle happened across a book called The Intelligent Network. -Thumbing through it at random, Nagle came across an entire chapter -meticulously detailing the workings of E911 police emergency systems. -This extensive text was being sold openly, and yet in Illinois -a young man was in danger of going to prison for publishing -a thin six-page document about 911 service. - -Nagle made an ironic comment to this effect in Telecom Digest. -From there, Nagle was put in touch with Mitch Kapor, -and then with Neidorf's lawyers. - -Sheldon Zenner was delighted to find a computer telecommunications expert -willing to speak up for Neidorf, one who was not a wacky teenage "hacker." -Nagle was fluent, mature, and respectable; he'd once had a federal -security clearance. - -Nagle was asked to fly to Illinois to join the defense team. - -Having joined the defense as an expert witness, Nagle read the entire -E911 Document for himself. He made his own judgement about its potential -for menace. - -The time has now come for you yourself, the reader, to have a look -at the E911 Document. This six-page piece of work was the pretext -for a federal prosecution that could have sent an electronic publisher -to prison for thirty, or even sixty, years. It was the pretext -for the search and seizure of Steve Jackson Games, a legitimate publisher -of printed books. It was also the formal pretext for the search -and seizure of the Mentor's bulletin board, "Phoenix Project," -and for the raid on the home of Erik Bloodaxe. It also had much -to do with the seizure of Richard Andrews' Jolnet node -and the shutdown of Charles Boykin's AT&T node. -The E911 Document was the single most important piece -of evidence in the Hacker Crackdown. There can be no real -and legitimate substitute for the Document itself. - - -==Phrack Inc.== - -Volume Two, Issue 24, File 5 of 13 - -Control Office Administration -Of Enhanced 911 Services For -Special Services and Account Centers - -by the Eavesdropper - -March, 1988 - - -Description of Service -~~~~~~~~~~~~~~~~~~~~~ -The control office for Emergency 911 service is assigned in -accordance with the existing standard guidelines to one of -the following centers: - -o Special Services Center (SSC) -o Major Accounts Center (MAC) -o Serving Test Center (STC) -o Toll Control Center (TCC) - -The SSC/MAC designation is used in this document interchangeably -for any of these four centers. The Special Services Centers (SSCs) -or Major Account Centers (MACs) have been designated as the trouble -reporting contact for all E911 customer (PSAP) reported troubles. -Subscribers who have trouble on an E911 call will continue -to contact local repair service (CRSAB) who will refer the -trouble to the SSC/MAC, when appropriate. - -Due to the critical nature of E911 service, the control -and timely repair of troubles is demanded. As the primary -E911 customer contact, the SSC/MAC is in the unique position -to monitor the status of the trouble and insure its resolution. - -System Overview -~~~~~~~~~~~~~~ -The number 911 is intended as a nationwide universal -telephone number which provides the public with direct -access to a Public Safety Answering Point (PSAP). A PSAP -is also referred to as an Emergency Service Bureau (ESB). -A PSAP is an agency or facility which is authorized by a -municipality to receive and respond to police, fire and/or -ambulance services. One or more attendants are located -at the PSAP facilities to receive and handle calls of an -emergency nature in accordance with the local municipal -requirements. - -An important advantage of E911 emergency service is -improved (reduced) response times for emergency -services. Also close coordination among agencies -providing various emergency services is a valuable -capability provided by E911 service. - -1A ESS is used as the tandem office for the E911 network to -route all 911 calls to the correct (primary) PSAP designated -to serve the calling station. The E911 feature was -developed primarily to provide routing to the correct PSAP -for all 911 calls. Selective routing allows a 911 call -originated from a particular station located in a particular -district, zone, or town, to be routed to the primary PSAP -designated to serve that customer station regardless of -wire center boundaries. Thus, selective routing eliminates -the problem of wire center boundaries not coinciding with -district or other political boundaries. - -The services available with the E911 feature include: - -Forced Disconnect Default Routing -Alternative Routing Night Service -Selective Routing Automatic Number -Identification (ANI) -Selective Transfer Automatic Location -Identification (ALI) - - -Preservice/Installation Guidelines -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -When a contract for an E911 system has been signed, it is -the responsibility of Network Marketing to establish an -implementation/cutover committee which should include -a representative from the SSC/MAC. Duties of the E911 -Implementation Team include coordination of all phases -of the E911 system deployment and the formation of an -on-going E911 maintenance subcommittee. - -Marketing is responsible for providing the following -customer specific information to the SSC/MAC prior to -the start of call through testing: - -o All PSAP's (name, address, local contact) -o All PSAP circuit ID's -o 1004 911 service request including PSAP details on each PSAP - (1004 Section K, L, M) -o Network configuration -o Any vendor information (name, telephone number, equipment) - -The SSC/MAC needs to know if the equipment and sets -at the PSAP are maintained by the BOCs, an independent -company, or an outside vendor, or any combination. -This information is then entered on the PSAP profile sheets -and reviewed quarterly for changes, additions and deletions. - -Marketing will secure the Major Account Number (MAN) -and provide this number to Corporate Communications -so that the initial issue of the service orders carry -the MAN and can be tracked by the SSC/MAC via CORDNET. -PSAP circuits are official services by definition. - -All service orders required for the installation of the E911 -system should include the MAN assigned to the city/county -which has purchased the system. - -In accordance with the basic SSC/MAC strategy for provisioning, -the SSC/MAC will be Overall Control Office (OCO) for all Node -to PSAP circuits (official services) and any other services -for this customer. Training must be scheduled for all SSC/MAC -involved personnel during the pre-service stage of the project. - -The E911 Implementation Team will form the on-going -maintenance subcommittee prior to the initial -implementation of the E911 system. This sub-committee -will establish post implementation quality assurance -procedures to ensure that the E911 system continues to -provide quality service to the customer. -Customer/Company training, trouble reporting interfaces -for the customer, telephone company and any involved -independent telephone companies needs to be addressed -and implemented prior to E911 cutover. These functions -can be best addressed by the formation of a sub- -committee of the E911 Implementation Team to set up -guidelines for and to secure service commitments of -interfacing organizations. A SSC/MAC supervisor should -chair this subcommittee and include the following -organizations: - -1) Switching Control Center - - E911 translations - - Trunking - - End office and Tandem office hardware/software -2) Recent Change Memory Administration Center - - Daily RC update activity for TN/ESN translations - - Processes validity errors and rejects -3) Line and Number Administration - - Verification of TN/ESN translations -4) Special Service Center/Major Account Center - - Single point of contact for all PSAP and Node to host troubles - - Logs, tracks & statusing of all trouble reports - - Trouble referral, follow up, and escalation - - Customer notification of status and restoration - - Analyzation of "chronic" troubles - - Testing, installation and maintenance of E911 circuits -5) Installation and Maintenance (SSIM/I&M) - - Repair and maintenance of PSAP equipment and Telco owned sets -6) Minicomputer Maintenance Operations Center - - E911 circuit maintenance (where applicable) -7) Area Maintenance Engineer - - Technical assistance on voice (CO-PSAP) network related E911 troubles - - -Maintenance Guidelines -~~~~~~~~~~~~~~~~~~~~~ -The CCNC will test the Node circuit from the 202T at the -Host site to the 202T at the Node site. Since Host to Node -(CCNC to MMOC) circuits are official company services, -the CCNC will refer all Node circuit troubles to the -SSC/MAC. The SSC/MAC is responsible for the testing -and follow up to restoration of these circuit troubles. - -Although Node to PSAP circuit are official services, the -MMOC will refer PSAP circuit troubles to the appropriate -SSC/MAC. The SSC/MAC is responsible for testing and -follow up to restoration of PSAP circuit troubles. - -The SSC/MAC will also receive reports from -CRSAB/IMC(s) on subscriber 911 troubles when they are -not line troubles. The SSC/MAC is responsible for testing -and restoration of these troubles. - -Maintenance responsibilities are as follows: - -SCC@ Voice Network (ANI to PSAP) -@SCC responsible for tandem switch - -SSIM/I&M PSAP Equipment (Modems, CIU's, sets) -Vendor PSAP Equipment (when CPE) -SSC/MAC PSAP to Node circuits, and tandem to - PSAP voice circuits (EMNT) -MMOC Node site (Modems, cables, etc) - -Note: All above work groups are required to resolve troubles -by interfacing with appropriate work groups for resolution. - -The Switching Control Center (SCC) is responsible for -E911/1AESS translations in tandem central offices. -These translations route E911 calls, selective transfer, -default routing, speed calling, etc., for each PSAP. -The SCC is also responsible for troubleshooting on -the voice network (call originating to end office tandem equipment). - -For example, ANI failures in the originating offices would -be a responsibility of the SCC. - -Recent Change Memory Administration Center (RCMAC) performs -the daily tandem translation updates (recent change) -for routing of individual telephone numbers. - -Recent changes are generated from service order activity -(new service, address changes, etc.) and compiled into -a daily file by the E911 Center (ALI/DMS E911 Computer). - -SSIM/I&M is responsible for the installation and repair of -PSAP equipment. PSAP equipment includes ANI Controller, -ALI Controller, data sets, cables, sets, and other peripheral -equipment that is not vendor owned. SSIM/I&M is responsible -for establishing maintenance test kits, complete with spare parts -for PSAP maintenance. This includes test gear, data sets, -and ANI/ALI Controller parts. - -Special Services Center (SSC) or Major Account Center -(MAC) serves as the trouble reporting contact for all -(PSAP) troubles reported by customer. The SSC/MAC -refers troubles to proper organizations for handling and -tracks status of troubles, escalating when necessary. -The SSC/MAC will close out troubles with customer. -The SSC/MAC will analyze all troubles and tracks "chronic" -PSAP troubles. - -Corporate Communications Network Center (CCNC) will -test and refer troubles on all node to host circuits. -All E911 circuits are classified as official company property. - -The Minicomputer Maintenance Operations Center -(MMOC) maintains the E911 (ALI/DMS) computer -hardware at the Host site. This MMOC is also responsible -for monitoring the system and reporting certain PSAP -and system problems to the local MMOC's, SCC's or -SSC/MAC's. The MMOC personnel also operate software -programs that maintain the TN data base under the -direction of the E911 Center. The maintenance of the -NODE computer (the interface between the PSAP and the -ALI/DMS computer) is a function of the MMOC at the -NODE site. The MMOC's at the NODE sites may also be -involved in the testing of NODE to Host circuits. -The MMOC will also assist on Host to PSAP and data network -related troubles not resolved through standard trouble -clearing procedures. - -Installation And Maintenance Center (IMC) is responsible -for referral of E911 subscriber troubles that are not subscriber -line problems. - -E911 Center - Performs the role of System Administration -and is responsible for overall operation of the E911 -computer software. The E911 Center does A-Z trouble -analysis and provides statistical information on the -performance of the system. - -This analysis includes processing PSAP inquiries (trouble -reports) and referral of network troubles. The E911 Center -also performs daily processing of tandem recent change -and provides information to the RCMAC for tandem input. -The E911 Center is responsible for daily processing -of the ALI/DMS computer data base and provides error files, -etc. to the Customer Services department for investigation and correction. -The E911 Center participates in all system implementations and on-going -maintenance effort and assists in the development of procedures, -training and education of information to all groups. - -Any group receiving a 911 trouble from the SSC/MAC should -close out the trouble with the SSC/MAC or provide a status -if the trouble has been referred to another group. -This will allow the SSC/MAC to provide a status back -to the customer or escalate as appropriate. - -Any group receiving a trouble from the Host site (MMOC -or CCNC) should close the trouble back to that group. - -The MMOC should notify the appropriate SSC/MAC -when the Host, Node, or all Node circuits are down so that -the SSC/MAC can reply to customer reports that may be -called in by the PSAPs. This will eliminate duplicate -reporting of troubles. On complete outages the MMOC -will follow escalation procedures for a Node after two (2) -hours and for a PSAP after four (4) hours. Additionally the -MMOC will notify the appropriate SSC/MAC when the -Host, Node, or all Node circuits are down. - -The PSAP will call the SSC/MAC to report E911 troubles. -The person reporting the E911 trouble may not have a -circuit I.D. and will therefore report the PSAP name and -address. Many PSAP troubles are not circuit specific. In -those instances where the caller cannot provide a circuit -I.D., the SSC/MAC will be required to determine the -circuit I.D. using the PSAP profile. Under no circumstances -will the SSC/MAC Center refuse to take the trouble. -The E911 trouble should be handled as quickly as possible, -with the SSC/MAC providing as much assistance as -possible while taking the trouble report from the caller. - -The SSC/MAC will screen/test the trouble to determine the -appropriate handoff organization based on the following criteria: - -PSAP equipment problem: SSIM/I&M -Circuit problem: SSC/MAC -Voice network problem: SCC (report trunk group number) -Problem affecting multiple PSAPs (No ALI report from -all PSAPs): Contact the MMOC to check for NODE or -Host computer problems before further testing. - -The SSC/MAC will track the status of reported troubles -and escalate as appropriate. The SSC/MAC will close out -customer/company reports with the initiating contact. -Groups with specific maintenance responsibilities, -defined above, will investigate "chronic" troubles upon -request from the SSC/MAC and the ongoing maintenance subcommittee. - -All "out of service" E911 troubles are priority one type reports. -One link down to a PSAP is considered a priority one trouble -and should be handled as if the PSAP was isolated. - -The PSAP will report troubles with the ANI controller, ALI -controller or set equipment to the SSC/MAC. - -NO ANI: Where the PSAP reports NO ANI (digital -display screen is blank) ask if this condition exists on all -screens and on all calls. It is important to differentiate -between blank screens and screens displaying 911-00XX, -or all zeroes. - -When the PSAP reports all screens on all calls, ask if there -is any voice contact with callers. If there is no voice -contact the trouble should be referred to the SCC -immediately since 911 calls are not getting through which -may require alternate routing of calls to another PSAP. - -When the PSAP reports this condition on all screens -but not all calls and has voice contact with callers, -the report should be referred to SSIM/I&M for dispatch. -The SSC/MAC should verify with the SCC that ANI -is pulsing before dispatching SSIM. - -When the PSAP reports this condition on one screen for -all calls (others work fine) the trouble should be referred -to SSIM/I&M for dispatch, because the trouble is isolated to -one piece of equipment at the customer premise. - -An ANI failure (i.e. all zeroes) indicates that the ANI has -not been received by the PSAP from the tandem office or -was lost by the PSAP ANI controller. The PSAP may -receive "02" alarms which can be caused by the ANI -controller logging more than three all zero failures on the -same trunk. The PSAP has been instructed to report this -condition to the SSC/MAC since it could indicate an -equipment trouble at the PSAP which might be affecting -all subscribers calling into the PSAP. When all zeroes are -being received on all calls or "02" alarms continue, a tester -should analyze the condition to determine the appropriate -action to be taken. The tester must perform cooperative -testing with the SCC when there appears to be a problem -on the Tandem-PSAP trunks before requesting dispatch. - -When an occasional all zero condition is reported, -the SSC/MAC should dispatch SSIM/I&M to routine -equipment on a "chronic" troublesweep. - -The PSAPs are instructed to report incidental ANI failures -to the BOC on a PSAP inquiry trouble ticket (paper) that -is sent to the Customer Services E911 group and forwarded -to E911 center when required. This usually involves only a -particular telephone number and is not a condition that -would require a report to the SSC/MAC. Multiple ANI -failures which our from the same end office (XX denotes -end office), indicate a hard trouble condition may exist -in the end office or end office tandem trunks. The PSAP will -report this type of condition to the SSC/MAC and the -SSC/MAC should refer the report to the SCC responsible -for the tandem office. NOTE: XX is the ESCO (Emergency -Service Number) associated with the incoming 911 trunks -into the tandem. It is important that the C/MAC tell the -SCC what is displayed at the PSAP (i.e. 911-0011) which -indicates to the SCC which end office is in trouble. - -Note: It is essential that the PSAP fill out inquiry form -on every ANI failure. - -The PSAP will report a trouble any time an address is not -received on an address display (screen blank) E911 call. -(If a record is not in the 911 data base or an ANI failure -is encountered, the screen will provide a display noticing -such condition). The SSC/MAC should verify with the PSAP -whether the NO ALI condition is on one screen or all screens. - -When the condition is on one screen (other screens -receive ALI information) the SSC/MAC will request -SSIM/I&M to dispatch. - -If no screens are receiving ALI information, there is usually -a circuit trouble between the PSAP and the Host computer. -The SSC/MAC should test the trouble and refer for restoral. - -Note: If the SSC/MAC receives calls from multiple -PSAP's, all of which are receiving NO ALI, there is a -problem with the Node or Node to Host circuits or the -Host computer itself. Before referring the trouble the -SSC/MAC should call the MMOC to inquire if the Node -or Host is in trouble. - -Alarm conditions on the ANI controller digital display at -the PSAP are to be reported by the PSAP's. These alarms -can indicate various trouble conditions so the SSC/MAC -should ask the PSAP if any portion of the E911 system -is not functioning properly. - -The SSC/MAC should verify with the PSAP attendant that -the equipment's primary function is answering E911 calls. -If it is, the SSC/MAC should request a dispatch SSIM/I&M. -If the equipment is not primarily used for E911, -then the SSC/MAC should advise PSAP to contact their CPE vendor. - -Note: These troubles can be quite confusing when the -PSAP has vendor equipment mixed in with equipment -that the BOC maintains. The Marketing representative -should provide the SSC/MAC information concerning any -unusual or exception items where the PSAP should -contact their vendor. This information should be included -in the PSAP profile sheets. - -ANI or ALI controller down: When the host computer sees -the PSAP equipment down and it does not come back up, -the MMOC will report the trouble to the SSC/MAC; -the equipment is down at the PSAP, a dispatch will be required. - -PSAP link (circuit) down: The MMOC will provide the -SSC/MAC with the circuit ID that the Host computer -indicates in trouble. Although each PSAP has two circuits, -when either circuit is down the condition must be treated -as an emergency since failure of the second circuit will -cause the PSAP to be isolated. - -Any problems that the MMOC identifies from the Node -location to the Host computer will be handled directly -with the appropriate MMOC(s)/CCNC. - -Note: The customer will call only when a problem is -apparent to the PSAP. When only one circuit is down to -the PSAP, the customer may not be aware there is a -trouble, even though there is one link down, -notification should appear on the PSAP screen. -Troubles called into the SSC/MAC from the MMOC -or other company employee should not be closed out -by calling the PSAP since it may result in the -customer responding that they do not have a trouble. -These reports can only be closed out by receiving -information that the trouble was fixed and by checking -with the company employee that reported the trouble. -The MMOC personnel will be able to verify that the -trouble has cleared by reviewing a printout from the host. - -When the CRSAB receives a subscriber complaint -(i.e., cannot dial 911) the RSA should obtain as much -information as possible while the customer is on the line. - -For example, what happened when the subscriber dialed 911? -The report is automatically directed to the IMC for subscriber line testing. -When no line trouble is found, the IMC will refer the trouble condition -to the SSC/MAC. The SSC/MAC will contact Customer Services E911 Group -and verify that the subscriber should be able to call 911 and obtain the ESN. -The SSC/MAC will verify the ESN via 2SCCS. When both verifications match, -the SSC/MAC will refer the report to the SCC responsible for the 911 tandem -office for investigation and resolution. The MAC is responsible for tracking -the trouble and informing the IMC when it is resolved. - - -For more information, please refer to E911 Glossary of Terms. -End of Phrack File -_____________________________________ - - -The reader is forgiven if he or she was entirely unable to read -this document. John Perry Barlow had a great deal of fun at its expense, -in "Crime and Puzzlement:" "Bureaucrat-ese of surpassing opacity. . . . -To read the whole thing straight through without entering coma requires -either a machine or a human who has too much practice thinking like one. -Anyone who can understand it fully and fluidly had altered his consciousness -beyond the ability to ever again read Blake, Whitman, or Tolstoy. . . . -the document contains little of interest to anyone who is not a student -of advanced organizational sclerosis." - -With the Document itself to hand, however, exactly as it was published -(in its six-page edited form) in Phrack, the reader may be able to verify -a few statements of fact about its nature. First, there is no software, -no computer code, in the Document. It is not computer-programming language -like FORTRAN or C++, it is English; all the sentences have nouns and verbs -and punctuation. It does not explain how to break into the E911 system. -It does not suggest ways to destroy or damage the E911 system. - -There are no access codes in the Document. There are no computer passwords. -It does not explain how to steal long distance service. It does not explain -how to break in to telco switching stations. There is nothing in it about -using a personal computer or a modem for any purpose at all, good or bad. - -Close study will reveal that this document is not about machinery. -The E911 Document is about ADMINISTRATION. It describes how one creates -and administers certain units of telco bureaucracy: -Special Service Centers and Major Account Centers (SSC/MAC). -It describes how these centers should distribute responsibility -for the E911 service, to other units of telco bureaucracy, -in a chain of command, a formal hierarchy. It describes -who answers customer complaints, who screens calls, -who reports equipment failures, who answers those reports, -who handles maintenance, who chairs subcommittees, -who gives orders, who follows orders, WHO tells WHOM what to do. -The Document is not a "roadmap" to computers. -The Document is a roadmap to PEOPLE. - -As an aid to breaking into computer systems, the Document is USELESS. -As an aid to harassing and deceiving telco people, however, the Document -might prove handy (especially with its Glossary, which I have not included). -An intense and protracted study of this Document and its Glossary, -combined with many other such documents, might teach one to speak like -a telco employee. And telco people live by SPEECH--they live by phone -communication. If you can mimic their language over the phone, -you can "social-engineer" them. If you can con telco people, you can -wreak havoc among them. You can force them to no longer trust one another; -you can break the telephonic ties that bind their community; you can make -them paranoid. And people will fight harder to defend their community -than they will fight to defend their individual selves. - -This was the genuine, gut-level threat posed by Phrack magazine. -The real struggle was over the control of telco language, -the control of telco knowledge. It was a struggle to defend the social -"membrane of differentiation" that forms the walls of the telco -community's ivory tower --the special jargon that allows telco -professionals to recognize one another, and to exclude charlatans, -thieves, and upstarts. And the prosecution brought out this fact. -They repeatedly made reference to the threat posed to telco professionals -by hackers using "social engineering." - -However, Craig Neidorf was not on trial for learning to speak like -a professional telecommunications expert. Craig Neidorf was on trial -for access device fraud and transportation of stolen property. -He was on trial for stealing a document that was purportedly -highly sensitive and purportedly worth tens of thousands of dollars. - -# - -John Nagle read the E911 Document. He drew his own conclusions. -And he presented Zenner and his defense team with an overflowing box -of similar material, drawn mostly from Stanford University's -engineering libraries. During the trial, the defense team--Zenner, -half-a-dozen other attorneys, Nagle, Neidorf, and computer-security -expert Dorothy Denning, all pored over the E911 Document line-by-line. - -On the afternoon of July 25, 1990, Zenner began to cross-examine -a woman named Billie Williams, a service manager for Southern Bell -in Atlanta. Ms. Williams had been responsible for the E911 Document. -(She was not its author--its original "author" was a Southern Bell -staff manager named Richard Helms. However, Mr. Helms should not bear -the entire blame; many telco staff people and maintenance personnel -had amended the Document. It had not been so much "written" by a -single author, as built by committee out of concrete-blocks of jargon.) - -Ms. Williams had been called as a witness for the prosecution, -and had gamely tried to explain the basic technical structure -of the E911 system, aided by charts. - -Now it was Zenner's turn. He first established that the -"proprietary stamp" that BellSouth had used on the E911 Document -was stamped on EVERY SINGLE DOCUMENT that BellSouth wrote-- -THOUSANDS of documents. "We do not publish anything other -than for our own company," Ms. Williams explained. -"Any company document of this nature is considered proprietary." -Nobody was in charge of singling out special high-security publications -for special high-security protection. They were ALL special, -no matter how trivial, no matter what their subject matter-- -the stamp was put on as soon as any document was written, -and the stamp was never removed. - -Zenner now asked whether the charts she had been using to explain -the mechanics of E911 system were "proprietary," too. -Were they PUBLIC INFORMATION, these charts, all about PSAPs, -ALIs, nodes, local end switches? Could he take the charts out -in the street and show them to anybody, "without violating -some proprietary notion that BellSouth has?" - -Ms Williams showed some confusion, but finally areed that the charts were, -in fact, public. - -"But isn't this what you said was basically what appeared in Phrack?" - -Ms. Williams denied this. - -Zenner now pointed out that the E911 Document as published in Phrack -was only half the size of the original E911 Document (as Prophet -had purloined it). Half of it had been deleted--edited by Neidorf. - -Ms. Williams countered that "Most of the information that is -in the text file is redundant." - -Zenner continued to probe. Exactly what bits of knowledge in the Document -were, in fact, unknown to the public? Locations of E911 computers? -Phone numbers for telco personnel? Ongoing maintenance subcommittees? -Hadn't Neidorf removed much of this? - -Then he pounced. "Are you familiar with Bellcore Technical Reference -Document TR-TSY-000350?" It was, Zenner explained, officially titled -"E911 Public Safety Answering Point Interface Between 1-1AESS Switch -and Customer Premises Equipment." It contained highly detailed -and specific technical information about the E911 System. -It was published by Bellcore and publicly available for about $20. - -He showed the witness a Bellcore catalog which listed thousands -of documents from Bellcore and from all the Baby Bells, BellSouth included. -The catalog, Zenner pointed out, was free. Anyone with a credit card -could call the Bellcore toll-free 800 number and simply order any -of these documents, which would be shipped to any customer without question. -Including, for instance, "BellSouth E911 Service Interfaces to -Customer Premises Equipment at a Public Safety Answering Point." - -Zenner gave the witness a copy of "BellSouth E911 Service Interfaces," -which cost, as he pointed out, $13, straight from the catalog. -"Look at it carefully," he urged Ms. Williams, "and tell me -if it doesn't contain about twice as much detailed information -about the E911 system of BellSouth than appeared anywhere in Phrack." - -"You want me to. . . ." Ms. Williams trailed off. "I don't understand." - -"Take a careful look," Zenner persisted. "Take a look at that document, -and tell me when you're done looking at it if, indeed, it doesn't contain -much more detailed information about the E911 system than appeared in Phrack." - -"Phrack wasn't taken from this," Ms. Williams said. - -"Excuse me?" said Zenner. - -"Phrack wasn't taken from this." - -"I can't hear you," Zenner said. - -"Phrack was not taken from this document. I don't understand -your question to me." - -"I guess you don't," Zenner said. - -At this point, the prosecution's case had been gutshot. -Ms. Williams was distressed. Her confusion was quite genuine. -Phrack had not been taken from any publicly available Bellcore document. -Phrack's E911 Document had been stolen from her own company's computers, -from her own company's text files, that her own colleagues had written, -and revised, with much labor. - -But the "value" of the Document had been blown to smithereens. -It wasn't worth eighty grand. According to Bellcore it was worth -thirteen bucks. And the looming menace that it supposedly posed -had been reduced in instants to a scarecrow. Bellcore itself -was selling material far more detailed and "dangerous," -to anybody with a credit card and a phone. - -Actually, Bellcore was not giving this information to just anybody. -They gave it to ANYBODY WHO ASKED, but not many did ask. -Not many people knew that Bellcore had a free catalog and an 800 number. -John Nagle knew, but certainly the average teenage phreak didn't know. -"Tuc," a friend of Neidorf's and sometime Phrack contributor, knew, -and Tuc had been very helpful to the defense, behind the scenes. -But the Legion of Doom didn't know--otherwise, they would never -have wasted so much time raiding dumpsters. Cook didn't know. -Foley didn't know. Kluepfel didn't know. The right hand -of Bellcore knew not what the left hand was doing. The right -hand was battering hackers without mercy, while the left hand -was distributing Bellcore's intellectual property to anybody -who was interested in telephone technical trivia--apparently, -a pathetic few. - -The digital underground was so amateurish and poorly organized -that they had never discovered this heap of unguarded riches. -The ivory tower of the telcos was so wrapped-up in the fog -of its own technical obscurity that it had left all the -windows open and flung open the doors. No one had even noticed. - -Zenner sank another nail in the coffin. He produced a printed issue -of Telephone Engineer & Management, a prominent industry journal -that comes out twice a month and costs $27 a year. This particular issue -of TE&M, called "Update on 911," featured a galaxy of technical details -on 911 service and a glossary far more extensive than Phrack's. - -The trial rumbled on, somehow, through its own momentum. -Tim Foley testified about his interrogations of Neidorf. -Neidorf's written admission that he had known the E911 Document -was pilfered was officially read into the court record. - -An interesting side issue came up: "Terminus" had once passed Neidorf -a piece of UNIX AT&T software, a log-in sequence, that had been cunningly -altered so that it could trap passwords. The UNIX software itself was -illegally copied AT&T property, and the alterations "Terminus" had made to it, -had transformed it into a device for facilitating computer break-ins. Terminus -himself would eventually plead guilty to theft of this piece of software, -and the Chicago group would send Terminus to prison for it. But it was -of dubious relevance in the Neidorf case. Neidorf hadn't written the program. -He wasn't accused of ever having used it. And Neidorf wasn't being charged -with software theft or owning a password trapper. - -On the next day, Zenner took the offensive. The civil libertarians -now had their own arcane, untried legal weaponry to launch into action-- -the Electronic Communications Privacy Act of 1986, 18 US Code, -Section 2701 et seq. Section 2701 makes it a crime to intentionally -access without authorization a facility in which an electronic communication -service is provided--it is, at heart, an anti-bugging and anti-tapping law, -intended to carry the traditional protections of telephones into other -electronic channels of communication. While providing penalties for amateur -snoops, however, Section 2703 of the ECPA also lays some formal difficulties -on the bugging and tapping activities of police. - -The Secret Service, in the person of Tim Foley, had served Richard Andrews -with a federal grand jury subpoena, in their pursuit of Prophet, -the E911 Document, and the Terminus software ring. But according to -the Electronic Communications Privacy Act, a "provider of remote -computing service" was legally entitled to "prior notice" from -the government if a subpoena was used. Richard Andrews and his -basement UNIX node, Jolnet, had not received any "prior notice." -Tim Foley had purportedly violated the ECPA and committed -an electronic crime! Zenner now sought the judge's permission -to cross-examine Foley on the topic of Foley's own electronic misdeeds. - -Cook argued that Richard Andrews' Jolnet was a privately owned -bulletin board, and not within the purview of ECPA. Judge Bua -granted the motion of the government to prevent cross-examination -on that point, and Zenner's offensive fizzled. This, however, -was the first direct assault on the legality of the actions -of the Computer Fraud and Abuse Task Force itself-- -the first suggestion that they themselves had broken the law, -and might, perhaps, be called to account. - -Zenner, in any case, did not really need the ECPA. -Instead, he grilled Foley on the glaring contradictions in -the supposed value of the E911 Document. He also brought up -the embarrassing fact that the supposedly red-hot E911 Document -had been sitting around for months, in Jolnet, with Kluepfel's knowledge, -while Kluepfel had done nothing about it. - -In the afternoon, the Prophet was brought in to testify -for the prosecution. (The Prophet, it will be recalled, -had also been indicted in the case as partner in a fraud -scheme with Neidorf.) In Atlanta, the Prophet had already -pled guilty to one charge of conspiracy, one charge of wire fraud -and one charge of interstate transportation of stolen property. -The wire fraud charge, and the stolen property charge, -were both directly based on the E911 Document. - -The twenty-year-old Prophet proved a sorry customer, -answering questions politely but in a barely audible mumble, -his voice trailing off at the ends of sentences. -He was constantly urged to speak up. - -Cook, examining Prophet, forced him to admit that -he had once had a "drug problem," abusing amphetamines, -marijuana, cocaine, and LSD. This may have established -to the jury that "hackers" are, or can be, seedy lowlife characters, -but it may have damaged Prophet's credibility somewhat. -Zenner later suggested that drugs might have damaged Prophet's memory. -The interesting fact also surfaced that Prophet had never -physically met Craig Neidorf. He didn't even know -Neidorf's last name--at least, not until the trial. - -Prophet confirmed the basic facts of his hacker career. -He was a member of the Legion of Doom. He had abused codes, -he had broken into switching stations and re-routed calls, -he had hung out on pirate bulletin boards. He had raided -the BellSouth AIMSX computer, copied the E911 Document, -stored it on Jolnet, mailed it to Neidorf. He and Neidorf -had edited it, and Neidorf had known where it came from. - -Zenner, however, had Prophet confirm that Neidorf was not a member -of the Legion of Doom, and had not urged Prophet to break into -BellSouth computers. Neidorf had never urged Prophet to defraud anyone, -or to steal anything. Prophet also admitted that he had never known Neidorf -to break in to any computer. Prophet said that no one in the Legion of Doom -considered Craig Neidorf a "hacker" at all. Neidorf was not a UNIX maven, -and simply lacked the necessary skill and ability to break into computers. -Neidorf just published a magazine. - -On Friday, July 27, 1990, the case against Neidorf collapsed. -Cook moved to dismiss the indictment, citing "information currently -available to us that was not available to us at the inception of the trial." -Judge Bua praised the prosecution for this action, which he described as -"very responsible," then dismissed a juror and declared a mistrial. - -Neidorf was a free man. His defense, however, had cost himself -and his family dearly. Months of his life had been consumed in anguish; -he had seen his closest friends shun him as a federal criminal. -He owed his lawyers over a hundred thousand dollars, despite -a generous payment to the defense by Mitch Kapor. - -Neidorf was not found innocent. The trial was simply dropped. -Nevertheless, on September 9, 1991, Judge Bua granted Neidorf's -motion for the "expungement and sealing" of his indictment record. -The United States Secret Service was ordered to delete and destroy -all fingerprints, photographs, and other records of arrest -or processing relating to Neidorf's indictment, including -their paper documents and their computer records. - -Neidorf went back to school, blazingly determined to become a lawyer. -Having seen the justice system at work, Neidorf lost much of his enthusiasm -for merely technical power. At this writing, Craig Neidorf is working -in Washington as a salaried researcher for the American Civil Liberties Union. - -# - -The outcome of the Neidorf trial changed the EFF -from voices-in-the-wilderness to the media darlings -of the new frontier. - -Legally speaking, the Neidorf case was not a sweeping triumph -for anyone concerned. No constitutional principles had been established. -The issues of "freedom of the press" for electronic publishers remained -in legal limbo. There were public misconceptions about the case. -Many people thought Neidorf had been found innocent and relieved -of all his legal debts by Kapor. The truth was that the government -had simply dropped the case, and Neidorf's family had gone deeply -into hock to support him. - -But the Neidorf case did provide a single, devastating, public sound-bite: -THE FEDS SAID IT WAS WORTH EIGHTY GRAND, AND IT WAS ONLY WORTH THIRTEEN BUCKS. - -This is the Neidorf case's single most memorable element. No serious report -of the case missed this particular element. Even cops could not read this -without a wince and a shake of the head. It left the public credibility -of the crackdown agents in tatters. - -The crackdown, in fact, continued, however. Those two charges -against Prophet, which had been based on the E911 Document, -were quietly forgotten at his sentencing--even though Prophet -had already pled guilty to them. Georgia federal prosecutors -strongly argued for jail time for the Atlanta Three, insisting on -"the need to send a message to the community," "the message that -hackers around the country need to hear." - -There was a great deal in their sentencing memorandum -about the awful things that various other hackers had done -(though the Atlanta Three themselves had not, in fact, -actually committed these crimes). There was also much -speculation about the awful things that the Atlanta Three -MIGHT have done and WERE CAPABLE of doing (even though -they had not, in fact, actually done them). -The prosecution's argument carried the day. -The Atlanta Three were sent to prison: -Urvile and Leftist both got 14 months each, -while Prophet (a second offender) got 21 months. - -The Atlanta Three were also assessed staggering fines as "restitution": -$233,000 each. BellSouth claimed that the defendants had "stolen" -"approximately $233,880 worth" of "proprietary computer access information"-- -specifically, $233,880 worth of computer passwords and connect addresses. -BellSouth's astonishing claim of the extreme value of its own computer -passwords and addresses was accepted at face value by the Georgia court. -Furthermore (as if to emphasize its theoretical nature) this enormous sum -was not divvied up among the Atlanta Three, but each of them had to pay -all of it. - -A striking aspect of the sentence was that the Atlanta Three were -specifically forbidden to use computers, except for work or under supervision. -Depriving hackers of home computers and modems makes some sense if one -considers hackers as "computer addicts," but EFF, filing an amicus brief -in the case, protested that this punishment was unconstitutional-- -it deprived the Atlanta Three of their rights of free association -and free expression through electronic media. - -Terminus, the "ultimate hacker," was finally sent to prison for a year -through the dogged efforts of the Chicago Task Force. His crime, -to which he pled guilty, was the transfer of the UNIX password trapper, -which was officially valued by AT&T at $77,000, a figure which aroused -intense skepticism among those familiar with UNIX "login.c" programs. - -The jailing of Terminus and the Atlanta Legionnaires of Doom, however, -did not cause the EFF any sense of embarrassment or defeat. -On the contrary, the civil libertarians were rapidly gathering strength. - -An early and potent supporter was Senator Patrick Leahy, -Democrat from Vermont, who had been a Senate sponsor -of the Electronic Communications Privacy Act. Even before -the Neidorf trial, Leahy had spoken out in defense of hacker-power -and freedom of the keyboard: "We cannot unduly inhibit the inquisitive -13-year-old who, if left to experiment today, may tomorrow develop -the telecommunications or computer technology to lead the United States -into the 21st century. He represents our future and our best hope -to remain a technologically competitive nation." - -It was a handsome statement, rendered perhaps rather more effective -by the fact that the crackdown raiders DID NOT HAVE any Senators -speaking out for THEM. On the contrary, their highly secretive -actions and tactics, all "sealed search warrants" here and -"confidential ongoing investigations" there, might have won -them a burst of glamorous publicity at first, but were crippling -them in the on-going propaganda war. Gail Thackeray was reduced -to unsupported bluster: "Some of these people who are loudest -on the bandwagon may just slink into the background," -she predicted in Newsweek--when all the facts came out, -and the cops were vindicated. - -But all the facts did not come out. Those facts that did, -were not very flattering. And the cops were not vindicated. -And Gail Thackeray lost her job. By the end of 1991, -William Cook had also left public employment. - -1990 had belonged to the crackdown, but by '91 its agents -were in severe disarray, and the libertarians were on a roll. -People were flocking to the cause. - -A particularly interesting ally had been Mike Godwin of Austin, Texas. -Godwin was an individual almost as difficult to describe as Barlow; -he had been editor of the student newspaper of the University of Texas, -and a computer salesman, and a programmer, and in 1990 was back -in law school, looking for a law degree. - -Godwin was also a bulletin board maven. He was very well-known -in the Austin board community under his handle "Johnny Mnemonic," -which he adopted from a cyberpunk science fiction story by William Gibson. -Godwin was an ardent cyberpunk science fiction fan. As a fellow Austinite -of similar age and similar interests, I myself had known Godwin socially -for many years. When William Gibson and myself had been writing our -collaborative SF novel, The Difference Engine, Godwin had been our -technical advisor in our effort to link our Apple word-processors -from Austin to Vancouver. Gibson and I were so pleased by his generous -expert help that we named a character in the novel "Michael Godwin" -in his honor. - -The handle "Mnemonic" suited Godwin very well. His erudition -and his mastery of trivia were impressive to the point of stupor; -his ardent curiosity seemed insatiable, and his desire to debate -and argue seemed the central drive of his life. Godwin had even -started his own Austin debating society, wryly known as the -"Dull Men's Club." In person, Godwin could be overwhelming; -a flypaper-brained polymath who could not seem to let any idea go. -On bulletin boards, however, Godwin's closely reasoned, -highly grammatical, erudite posts suited the medium well, -and he became a local board celebrity. - -Mike Godwin was the man most responsible for the public national exposure -of the Steve Jackson case. The Izenberg seizure in Austin had received -no press coverage at all. The March 1 raids on Mentor, Bloodaxe, and -Steve Jackson Games had received a brief front-page splash in the -front page of the Austin American-Statesman, but it was confused -and ill-informed: the warrants were sealed, and the Secret Service -wasn't talking. Steve Jackson seemed doomed to obscurity. -Jackson had not been arrested; he was not charged with any crime; -he was not on trial. He had lost some computers in an ongoing -investigation--so what? Jackson tried hard to attract attention -to the true extent of his plight, but he was drawing a blank; -no one in a position to help him seemed able to get a mental grip -on the issues. - -Godwin, however, was uniquely, almost magically, qualified -to carry Jackson's case to the outside world. Godwin was -a board enthusiast, a science fiction fan, a former journalist, -a computer salesman, a lawyer-to-be, and an Austinite. -Through a coincidence yet more amazing, in his last year -of law school Godwin had specialized in federal prosecutions -and criminal procedure. Acting entirely on his own, Godwin made -up a press packet which summarized the issues and provided useful -contacts for reporters. Godwin's behind-the-scenes effort -(which he carried out mostly to prove a point in a local board debate) -broke the story again in the Austin American-Statesman and then in Newsweek. - -Life was never the same for Mike Godwin after that. As he joined the growing -civil liberties debate on the Internet, it was obvious to all parties involved -that here was one guy who, in the midst of complete murk and confusion, -GENUINELY UNDERSTOOD EVERYTHING HE WAS TALKING ABOUT. The disparate elements -of Godwin's dilettantish existence suddenly fell together as neatly as -the facets of a Rubik's cube. - -When the time came to hire a full-time EFF staff attorney, -Godwin was the obvious choice. He took the Texas bar exam, -left Austin, moved to Cambridge, became a full-time, professional, -computer civil libertarian, and was soon touring the nation on behalf -of EFF, delivering well-received addresses on the issues to crowds -as disparate as academics, industrialists, science fiction fans, -and federal cops. - -Michael Godwin is currently the chief legal counsel of -the Electronic Frontier Foundation in Cambridge, Massachusetts. - -# - -Another early and influential participant in the controversy -was Dorothy Denning. Dr. Denning was unique among investigators -of the computer underground in that she did not enter the debate -with any set of politicized motives. She was a professional -cryptographer and computer security expert whose primary interest -in hackers was SCHOLARLY. She had a B.A. and M.A. in mathematics, -and a Ph.D. in computer science from Purdue. She had worked for SRI -International, the California think-tank that was also the home of -computer-security maven Donn Parker, and had authored an influential text -called Cryptography and Data Security. In 1990, Dr. Denning was working for -Digital Equipment Corporation in their Systems Reseach Center. Her husband, -Peter Denning, was also a computer security expert, working for NASA's -Research Institute for Advanced Computer Science. He had edited the -well-received Computers Under Attack: Intruders, Worms and Viruses. - -Dr. Denning took it upon herself to contact the digital underground, -more or less with an anthropological interest. There she discovered -that these computer-intruding hackers, who had been characterized -as unethical, irresponsible, and a serious danger to society, -did in fact have their own subculture and their own rules. -They were not particularly well-considered rules, but they were, -in fact, rules. Basically, they didn't take money and they -didn't break anything. - -Her dispassionate reports on her researches did a great deal -to influence serious-minded computer professionals--the sort -of people who merely rolled their eyes at the cyberspace -rhapsodies of a John Perry Barlow. - -For young hackers of the digital underground, meeting Dorothy Denning -was a genuinely mind-boggling experience. Here was this neatly coiffed, -conservatively dressed, dainty little personage, who reminded most -hackers of their moms or their aunts. And yet she was an IBM systems -programmer with profound expertise in computer architectures -and high-security information flow, who had personal friends -in the FBI and the National Security Agency. - -Dorothy Denning was a shining example of the American mathematical -intelligentsia, a genuinely brilliant person from the central ranks -of the computer-science elite. And here she was, gently questioning -twenty-year-old hairy-eyed phone-phreaks over the deeper ethical -implications of their behavior. - -Confronted by this genuinely nice lady, most hackers sat up very straight -and did their best to keep the anarchy-file stuff down to a faint whiff -of brimstone. Nevertheless, the hackers WERE in fact prepared to seriously -discuss serious issues with Dorothy Denning. They were willing to speak -the unspeakable and defend the indefensible, to blurt out their convictions -that information cannot be owned, that the databases of governments and large -corporations were a threat to the rights and privacy of individuals. - -Denning's articles made it clear to many that "hacking" -was not simple vandalism by some evil clique of psychotics. -"Hacking" was not an aberrant menace that could be charmed away -by ignoring it, or swept out of existence by jailing a few ringleaders. -Instead, "hacking" was symptomatic of a growing, primal struggle over -knowledge and power in the age of information. - -Denning pointed out that the attitude of hackers were at least partially -shared by forward-looking management theorists in the business community: -people like Peter Drucker and Tom Peters. Peter Drucker, in his book -The New Realities, had stated that "control of information by the government -is no longer possible. Indeed, information is now transnational. -Like money, it has no `fatherland.'" - -And management maven Tom Peters had chided large corporations for uptight, -proprietary attitudes in his bestseller, Thriving on Chaos: -"Information hoarding, especially by politically motivated, -power-seeking staffs, had been commonplace throughout American industry, -service and manufacturing alike. It will be an impossible -millstone aroung the neck of tomorrow's organizations." - -Dorothy Denning had shattered the social membrane of the -digital underground. She attended the Neidorf trial, -where she was prepared to testify for the defense as an expert witness. -She was a behind-the-scenes organizer of two of the most important -national meetings of the computer civil libertarians. Though not -a zealot of any description, she brought disparate elements of the -electronic community into a surprising and fruitful collusion. - -Dorothy Denning is currently the Chair of the Computer Science Department -at Georgetown University in Washington, DC. - -# - -There were many stellar figures in the civil libertarian community. -There's no question, however, that its single most influential figure -was Mitchell D. Kapor. Other people might have formal titles, -or governmental positions, have more experience with crime, -or with the law, or with the arcanities of computer security -or constitutional theory. But by 1991 Kapor had transcended -any such narrow role. Kapor had become "Mitch." - -Mitch had become the central civil-libertarian ad-hocrat. -Mitch had stood up first, he had spoken out loudly, directly, -vigorously and angrily, he had put his own reputation, -and his very considerable personal fortune, on the line. -By mid-'91 Kapor was the best-known advocate of his cause -and was known PERSONALLY by almost every single human being in America -with any direct influence on the question of civil liberties in cyberspace. -Mitch had built bridges, crossed voids, changed paradigms, forged metaphors, -made phone-calls and swapped business cards to such spectacular effect -that it had become impossible for anyone to take any action in the -"hacker question" without wondering what Mitch might think-- -and say--and tell his friends. - -The EFF had simply NETWORKED the situation into an entirely new status quo. -And in fact this had been EFF's deliberate strategy from the beginning. -Both Barlow and Kapor loathed bureaucracies and had deliberately -chosen to work almost entirely through the electronic spiderweb of -"valuable personal contacts." - -After a year of EFF, both Barlow and Kapor had every reason -to look back with satisfaction. EFF had established its own Internet node, -"eff.org," with a well-stocked electronic archive of documents on -electronic civil rights, privacy issues, and academic freedom. -EFF was also publishing EFFector, a quarterly printed journal, -as well as EFFector Online, an electronic newsletter with -over 1,200 subscribers. And EFF was thriving on the Well. - -EFF had a national headquarters in Cambridge and a full-time staff. -It had become a membership organization and was attracting -grass-roots support. It had also attracted the support -of some thirty civil-rights lawyers, ready and eager -to do pro bono work in defense of the Constitution in Cyberspace. - -EFF had lobbied successfully in Washington and in Massachusetts -to change state and federal legislation on computer networking. -Kapor in particular had become a veteran expert witness, -and had joined the Computer Science and Telecommunications Board -of the National Academy of Science and Engineering. - -EFF had sponsored meetings such as "Computers, Freedom and Privacy" -and the CPSR Roundtable. It had carried out a press offensive that, -in the words of EFFector, "has affected the climate of opinion about -computer networking and begun to reverse the slide into -`hacker hysteria' that was beginning to grip the nation." - -It had helped Craig Neidorf avoid prison. - -And, last but certainly not least, the Electronic Frontier Foundation -had filed a federal lawsuit in the name of Steve Jackson, -Steve Jackson Games Inc., and three users of the Illuminati -bulletin board system. The defendants were, and are, -the United States Secret Service, William Cook, Tim Foley, -Barbara Golden and Henry Kleupfel. - -The case, which is in pre-trial procedures in an Austin federal court -as of this writing, is a civil action for damages to redress -alleged violations of the First and Fourth Amendments to the -United States Constitution, as well as the Privacy Protection Act -of 1980 (42 USC 2000aa et seq.), and the Electronic Communications -Privacy Act (18 USC 2510 et seq and 2701 et seq). - -EFF had established that it had credibility. It had also established -that it had teeth. - -In the fall of 1991 I travelled to Massachusetts to speak personally -with Mitch Kapor. It was my final interview for this book. - -# - -The city of Boston has always been one of the major intellectual centers -of the American republic. It is a very old city by American standards, -a place of skyscrapers overshadowing seventeenth-century graveyards, -where the high-tech start-up companies of Route 128 co-exist with the -hand-wrought pre-industrial grace of "Old Ironsides," the USS CONSTITUTION. - -The Battle of Bunker Hill, one of the first and bitterest armed clashes -of the American Revolution, was fought in Boston's environs. Today there is -a monumental spire on Bunker Hill, visible throughout much of the city. -The willingness of the republican revolutionaries to take up arms and fire -on their oppressors has left a cultural legacy that two full centuries -have not effaced. Bunker Hill is still a potent center of American political -symbolism, and the Spirit of '76 is still a potent image for those who seek -to mold public opinion. - -Of course, not everyone who wraps himself in the flag is necessarily -a patriot. When I visited the spire in September 1991, it bore a huge, -badly-erased, spray-can grafitto around its bottom reading -"BRITS OUT--IRA PROVOS." Inside this hallowed edifice was -a glass-cased diorama of thousands of tiny toy soldiers, -rebels and redcoats, fighting and dying over the green hill, -the riverside marshes, the rebel trenchworks. Plaques indicated the -movement of troops, the shiftings of strategy. The Bunker Hill Monument -is occupied at its very center by the toy soldiers of a military -war-game simulation. - -The Boston metroplex is a place of great universities, -prominent among the Massachusetts Institute of Technology, -where the term "computer hacker" was first coined. The Hacker Crackdown -of 1990 might be interpreted as a political struggle among American cities: -traditional strongholds of longhair intellectual liberalism, -such as Boston, San Francisco, and Austin, versus the bare-knuckle -industrial pragmatism of Chicago and Phoenix (with Atlanta and New York -wrapped in internal struggle). - -The headquarters of the Electronic Frontier Foundation is on -155 Second Street in Cambridge, a Bostonian suburb north -of the River Charles. Second Street has weedy sidewalks of dented, -sagging brick and elderly cracked asphalt; large street-signs warn -"NO PARKING DURING DECLARED SNOW EMERGENCY." This is an old area -of modest manufacturing industries; the EFF is catecorner from the -Greene Rubber Company. EFF's building is two stories of red brick; -its large wooden windows feature gracefully arched tops and stone sills. - -The glass window beside the Second Street entrance bears three sheets -of neatly laser-printed paper, taped against the glass. They read: -ON Technology. EFF. KEI. - -"ON Technology" is Kapor's software company, which currently specializes -in "groupware" for the Apple Macintosh computer. "Groupware" is intended -to promote efficient social interaction among office-workers linked -by computers. ON Technology's most successful software products to date -are "Meeting Maker" and "Instant Update." - -"KEI" is Kapor Enterprises Inc., Kapor's personal holding company, -the commercial entity that formally controls his extensive investments -in other hardware and software corporations. - -"EFF" is a political action group--of a special sort. - -Inside, someone's bike has been chained to the handrails -of a modest flight of stairs. A wall of modish glass brick -separates this anteroom from the offices. Beyond the brick, -there's an alarm system mounted on the wall, a sleek, complex little -number that resembles a cross between a thermostat and a CD player. -Piled against the wall are box after box of a recent special issue -of Scientific American, "How to Work, Play, and Thrive in Cyberspace," -with extensive coverage of electronic networking techniques -and political issues, including an article by Kapor himself. -These boxes are addressed to Gerard Van der Leun, EFF's -Director of Communications, who will shortly mail those magazines -to every member of the EFF. - -The joint headquarters of EFF, KEI, and ON Technology, -which Kapor currently rents, is a modestly bustling place. -It's very much the same physical size as Steve Jackson's gaming company. -It's certainly a far cry from the gigantic gray steel-sided railway -shipping barn, on the Monsignor O'Brien Highway, that is owned -by Lotus Development Corporation. - -Lotus is, of course, the software giant that Mitchell Kapor founded -in the late 70s. The software program Kapor co-authored, -"Lotus 1-2-3," is still that company's most profitable product. -"Lotus 1-2-3" also bears a singular distinction in the -digital underground: it's probably the most pirated piece -of application software in world history. - -Kapor greets me cordially in his own office, down a hall. -Kapor, whose name is pronounced KAY-por, is in his early forties, -married and the father of two. He has a round face, high forehead, -straight nose, a slightly tousled mop of black hair peppered with gray. -His large brown eyes are wideset, reflective, one might almost say soulful. -He disdains ties, and commonly wears Hawaiian shirts and tropical prints, -not so much garish as simply cheerful and just that little bit anomalous. - -There is just the whiff of hacker brimstone about Mitch Kapor. -He may not have the hard-riding, hell-for-leather, guitar-strumming -charisma of his Wyoming colleague John Perry Barlow, but there's -something about the guy that still stops one short. He has the air -of the Eastern city dude in the bowler hat, the dreamy, -Longfellow-quoting poker shark who only HAPPENS to know -the exact mathematical odds against drawing to an inside straight. -Even among his computer-community colleagues, who are hardly known -for mental sluggishness, Kapor strikes one forcefully as a very -intelligent man. He speaks rapidly, with vigorous gestures, -his Boston accent sometimes slipping to the sharp nasal tang -of his youth in Long Island. - -Kapor, whose Kapor Family Foundation does much of his philanthropic work, -is a strong supporter of Boston's Computer Museum. Kapor's interest -in the history of his industry has brought him some remarkable curios, -such as the "byte" just outside his office door. This "byte"-- -eight digital bits--has been salvaged from the wreck of an -electronic computer of the pre-transistor age. It's a standing gunmetal -rack about the size of a small toaster-oven: with eight slots -of hand-soldered breadboarding featuring thumb-sized vacuum tubes. -If it fell off a table it could easily break your foot, -but it was state-of-the-art computation in the 1940s. -(It would take exactly 157,184 of these primordial toasters -to hold the first part of this book.) - -There's also a coiling, multicolored, scaly dragon that some -inspired techno-punk artist has cobbled up entirely out of transistors, -capacitors, and brightly plastic-coated wiring. - -Inside the office, Kapor excuses himself briefly to do a little -mouse-whizzing housekeeping on his personal Macintosh IIfx. -If its giant screen were an open window, an agile person -could climb through it without much trouble at all. -There's a coffee-cup at Kapor's elbow, a memento of his -recent trip to Eastern Europe, which has a black-and-white -stencilled photo and the legend CAPITALIST FOOLS TOUR. -It's Kapor, Barlow, and two California venture-capitalist luminaries -of their acquaintance, four windblown, grinning Baby Boomer -dudes in leather jackets, boots, denim, travel bags, -standing on airport tarmac somewhere behind the formerly Iron Curtain. -They look as if they're having the absolute time of their lives. - -Kapor is in a reminiscent mood. We talk a bit about his youth-- -high school days as a "math nerd," Saturdays attending Columbia University's -high-school science honors program, where he had his first experience -programming computers. IBM 1620s, in 1965 and '66. "I was very interested," -says Kapor, "and then I went off to college and got distracted by drugs sex -and rock and roll, like anybody with half a brain would have then!" -After college he was a progressive-rock DJ in Hartford, Connecticut, -for a couple of years. - -I ask him if he ever misses his rock and roll days--if he ever wished -he could go back to radio work. - -He shakes his head flatly. "I stopped thinking about going back -to be a DJ the day after Altamont." - -Kapor moved to Boston in 1974 and got a job programming mainframes in COBOL. -He hated it. He quit and became a teacher of transcendental meditation. -(It was Kapor's long flirtation with Eastern mysticism that gave the -world "Lotus.") - -In 1976 Kapor went to Switzerland, where the Transcendental Meditation -movement had rented a gigantic Victorian hotel in St-Moritz. It was -an all-male group--a hundred and twenty of them--determined upon -Enlightenment or Bust. Kapor had given the transcendant his best shot. -He was becoming disenchanted by "the nuttiness in the organization." -"They were teaching people to levitate," he says, staring at the floor. -His voice drops an octave, becomes flat. "THEY DON'T LEVITATE." - -Kapor chose Bust. He went back to the States and acquired a degree -in counselling psychology. He worked a while in a hospital, -couldn't stand that either. "My rep was," he says "a very bright kid -with a lot of potential who hasn't found himself. Almost thirty. -Sort of lost." - -Kapor was unemployed when he bought his first personal computer--an Apple II. -He sold his stereo to raise cash and drove to New Hampshire to avoid the -sales tax. - -"The day after I purchased it," Kapor tells me, "I was hanging out -in a computer store and I saw another guy, a man in his forties, -well-dressed guy, and eavesdropped on his conversation with the salesman. -He didn't know anything about computers. I'd had a year programming. -And I could program in BASIC. I'd taught myself. So I went up to him, -and I actually sold myself to him as a consultant." He pauses. -"I don't know where I got the nerve to do this. It was uncharacteristic. -I just said, `I think I can help you, I've been listening, -this is what you need to do and I think I can do it for you.' -And he took me on! He was my first client! I became a computer -consultant the first day after I bought the Apple II." - -Kapor had found his true vocation. He attracted more clients -for his consultant service, and started an Apple users' group. - -A friend of Kapor's, Eric Rosenfeld, a graduate student at MIT, -had a problem. He was doing a thesis on an arcane form of -financial statistics, but could not wedge himself into the crowded queue -for time on MIT's mainframes. (One might note at this point that if -Mr. Rosenfeld had dishonestly broken into the MIT mainframes, -Kapor himself might have never invented Lotus 1-2-3 and -the PC business might have been set back for years!) -Eric Rosenfeld did have an Apple II, however, -and he thought it might be possible to scale the problem down. -Kapor, as favor, wrote a program for him in BASIC that did the job. - -It then occurred to the two of them, out of the blue, -that it might be possible to SELL this program. -They marketed it themselves, in plastic baggies, -for about a hundred bucks a pop, mail order. -"This was a total cottage industry by a marginal consultant," -Kapor says proudly. "That's how I got started, honest to God." - -Rosenfeld, who later became a very prominent figure on Wall Street, -urged Kapor to go to MIT's business school for an MBA. -Kapor did seven months there, but never got his MBA. -He picked up some useful tools--mainly a firm grasp -of the principles of accounting--and, in his own words, -"learned to talk MBA." Then he dropped out and went to Silicon Valley. - -The inventors of VisiCalc, the Apple computer's premier business program, -had shown an interest in Mitch Kapor. Kapor worked diligently for them -for six months, got tired of California, and went back to Boston -where they had better bookstores. The VisiCalc group had made -the critical error of bringing in "professional management." -"That drove them into the ground," Kapor says. - -"Yeah, you don't hear a lot about VisiCalc these days," I muse. - -Kapor looks surprised. "Well, Lotus. . . we BOUGHT it." - -"Oh. You BOUGHT it?" - -"Yeah." - -"Sort of like the Bell System buying Western Union?" - -Kapor grins. "Yep! Yep! Yeah, exactly!" - -Mitch Kapor was not in full command of the destiny of himself -or his industry. The hottest software commodities of the early 1980s -were COMPUTER GAMES--the Atari seemed destined to enter every teenage home -in America. Kapor got into business software simply because he didn't have -any particular feeling for computer games. But he was supremely fast -on his feet, open to new ideas and inclined to trust his instincts. -And his instincts were good. He chose good people to deal with-- -gifted programmer Jonathan Sachs (the co-author of Lotus 1-2-3). -Financial wizard Eric Rosenfeld, canny Wall Street analyst -and venture capitalist Ben Rosen. Kapor was the founder and CEO of Lotus, -one of the most spectacularly successful business ventures of the -later twentieth century. - -He is now an extremely wealthy man. I ask him if he actually -knows how much money he has. - -"Yeah," he says. "Within a percent or two." - -How much does he actually have, then? - -He shakes his head. "A lot. A lot. Not something I talk about. -Issues of money and class are things that cut pretty close to the bone." - -I don't pry. It's beside the point. One might presume, impolitely, -that Kapor has at least forty million--that's what he got the year -he left Lotus. People who ought to know claim Kapor has about -a hundred and fifty million, give or take a market swing -in his stock holdings. If Kapor had stuck with Lotus, -as his colleague friend and rival Bill Gates has stuck -with his own software start-up, Microsoft, then Kapor -would likely have much the same fortune Gates has-- -somewhere in the neighborhood of three billion, -give or take a few hundred million. Mitch Kapor -has all the money he wants. Money has lost whatever charm -it ever held for him--probably not much in the first place. -When Lotus became too uptight, too bureaucratic, too far -from the true sources of his own satisfaction, Kapor walked. -He simply severed all connections with the company and went out the door. -It stunned everyone--except those who knew him best. - -Kapor has not had to strain his resources to wreak a thorough -transformation in cyberspace politics. In its first year, -EFF's budget was about a quarter of a million dollars. -Kapor is running EFF out of his pocket change. - -Kapor takes pains to tell me that he does not consider himself -a civil libertarian per se. He has spent quite some time -with true-blue civil libertarians lately, and there's a -political-correctness to them that bugs him. They seem -to him to spend entirely too much time in legal nitpicking -and not enough vigorously exercising civil rights in the -everyday real world. - -Kapor is an entrepreneur. Like all hackers, he prefers his involvements -direct, personal, and hands-on. "The fact that EFF has a node on the -Internet is a great thing. We're a publisher. We're a distributor -of information." Among the items the eff.org Internet node carries -is back issues of Phrack. They had an internal debate about that in EFF, -and finally decided to take the plunge. They might carry other -digital underground publications--but if they do, he says, -"we'll certainly carry Donn Parker, and anything Gail Thackeray -wants to put up. We'll turn it into a public library, that has -the whole spectrum of use. Evolve in the direction of people making up -their own minds." He grins. "We'll try to label all the editorials." - -Kapor is determined to tackle the technicalities of the Internet -in the service of the public interest. "The problem with being a node -on the Net today is that you've got to have a captive technical specialist. -We have Chris Davis around, for the care and feeding of the balky beast! -We couldn't do it ourselves!" - -He pauses. "So one direction in which technology has to evolve -is much more standardized units, that a non-technical person -can feel comfortable with. It's the same shift as from minicomputers to PCs. -I can see a future in which any person can have a Node on the Net. -Any person can be a publisher. It's better than the media we now have. -It's possible. We're working actively." - -Kapor is in his element now, fluent, thoroughly in command in his material. -"You go tell a hardware Internet hacker that everyone should have a node -on the Net," he says, "and the first thing they're going to say is, -`IP doesn't scale!'" ("IP" is the interface protocol for the Internet. -As it currently exists, the IP software is simply not capable of -indefinite expansion; it will run out of usable addresses, it will saturate.) -"The answer," Kapor says, "is: evolve the protocol! Get the smart people -together and figure out what to do. Do we add ID? Do we add new protocol? -Don't just say, WE CAN'T DO IT." - -Getting smart people together to figure out what to do is a skill -at which Kapor clearly excels. I counter that people on the Internet -rather enjoy their elite technical status, and don't seem particularly -anxious to democratize the Net. - -Kapor agrees, with a show of scorn. "I tell them that this is the snobbery -of the people on the Mayflower looking down their noses at the people -who came over ON THE SECOND BOAT! Just because they got here a year, -or five years, or ten years before everybody else, that doesn't give -them ownership of cyberspace! By what right?" - -I remark that the telcos are an electronic network, too, -and they seem to guard their specialized knowledge pretty closely. - -Kapor ripostes that the telcos and the Internet are entirely -different animals. "The Internet is an open system, -everything is published, everything gets argued about, -basically by anybody who can get in. Mostly, it's exclusive -and elitist just because it's so difficult. Let's make it easier to use." - -On the other hand, he allows with a swift change of emphasis, -the so-called elitists do have a point as well. "Before people start coming in, -who are new, who want to make suggestions, and criticize the Net as -`all screwed up'. . . . They should at least take the time to understand -the culture on its own terms. It has its own history--show some respect -for it. I'm a conservative, to that extent." - -The Internet is Kapor's paradigm for the future of telecommunications. -The Internet is decentralized, non-hierarchical, almost anarchic. -There are no bosses, no chain of command, no secret data. -If each node obeys the general interface standards, -there's simply no need for any central network authority. - -Wouldn't that spell the doom of AT&T as an institution? I ask. - -That prospect doesn't faze Kapor for a moment. "Their big advantage, -that they have now, is that they have all of the wiring. -But two things are happening. Anyone with right-of-way -is putting down fiber--Southern Pacific Railroad, -people like that--there's enormous `dark fiber' laid in." -("Dark Fiber" is fiber-optic cable, whose enormous capacity -so exceeds the demands of current usage that much of the -fiber still has no light-signals on it--it's still `dark,' -awaiting future use.) - -"The other thing that's happening is the local-loop stuff -is going to go wireless. Everyone from Bellcore to the cable TV -companies to AT&T wants to put in these things called -`personal communication systems.' So you could have local competition-- -you could have multiplicity of people, a bunch of neighborhoods, -sticking stuff up on poles. And a bunch of other people laying in dark fiber. -So what happens to the telephone companies? There's enormous pressure -on them from both sides. - -"The more I look at this, the more I believe that in a post-industrial, -digital world, the idea of regulated monopolies is bad. People will -look back on it and say that in the 19th and 20th centuries -the idea of public utilities was an okay compromise. -You needed one set of wires in the ground. It was too economically -inefficient, otherwise. And that meant one entity running it. -But now, with pieces being wireless--the connections are going -to be via high-level interfaces, not via wires. I mean, ULTIMATELY -there are going to be wires--but the wires are just a commodity. -Fiber, wireless. You no longer NEED a utility." - -Water utilities? Gas utilities? - -Of course we still need those, he agrees. "But when what you're moving -is information, instead of physical substances, then you can play by -a different set of rules. We're evolving those rules now! -Hopefully you can have a much more decentralized system, -and one in which there's more competition in the marketplace. - -"The role of government will be to make sure that nobody cheats. -The proverbial `level playing field.' A policy that prevents monopolization. -It should result in better service, lower prices, more choices, -and local empowerment." He smiles. "I'm very big on local empowerment." - -Kapor is a man with a vision. It's a very novel vision which he -and his allies are working out in considerable detail and with great energy. -Dark, cynical, morbid cyberpunk that I am, I cannot avoid considering -some of the darker implications of "decentralized, nonhierarchical, -locally empowered" networking. - -I remark that some pundits have suggested that electronic networking--faxes, -phones, small-scale photocopiers--played a strong role in dissolving -the power of centralized communism and causing the collapse of the Warsaw Pact. - -Socialism is totally discredited, says Kapor, fresh back from -the Eastern Bloc. The idea that faxes did it, all by themselves, -is rather wishful thinking. - -Has it occurred to him that electronic networking might corrode -America's industrial and political infrastructure to the point -where the whole thing becomes untenable, unworkable--and the old order -just collapses headlong, like in Eastern Europe? - -"No," Kapor says flatly. "I think that's extraordinarily unlikely. -In part, because ten or fifteen years ago, I had similar hopes -about personal computers--which utterly failed to materialize." -He grins wryly, then his eyes narrow. "I'm VERY opposed to techno-utopias. -Every time I see one, I either run away, or try to kill it." - -It dawns on me then that Mitch Kapor is not trying to -make the world safe for democracy. He certainly is not -trying to make it safe for anarchists or utopians-- -least of all for computer intruders or electronic rip-off artists. -What he really hopes to do is make the world safe for -future Mitch Kapors. This world of decentralized, small-scale nodes, -with instant global access for the best and brightest, -would be a perfect milieu for the shoestring attic capitalism -that made Mitch Kapor what he is today. - -Kapor is a very bright man. He has a rare combination -of visionary intensity with a strong practical streak. -The Board of the EFF: John Barlow, Jerry Berman of the ACLU, -Stewart Brand, John Gilmore, Steve Wozniak, and Esther Dyson, -the doyenne of East-West computer entrepreneurism--share his gift, -his vision, and his formidable networking talents. -They are people of the 1960s, winnowed-out by its turbulence -and rewarded with wealth and influence. They are some of the best -and the brightest that the electronic community has to offer. -But can they do it, in the real world? Or are they only dreaming? -They are so few. And there is so much against them. - -I leave Kapor and his networking employees struggling cheerfully -with the promising intricacies of their newly installed Macintosh -System 7 software. The next day is Saturday. EFF is closed. -I pay a few visits to points of interest downtown. - -One of them is the birthplace of the telephone. - -It's marked by a bronze plaque in a plinth of black-and-white speckled granite. It sits in the -plaza of the John F. Kennedy Federal Building, the very place where Kapor was -once fingerprinted by the FBI. - -The plaque has a bas-relief picture of Bell's original telephone. -"BIRTHPLACE OF THE TELEPHONE," it reads. "Here, on June 2, 1875, -Alexander Graham Bell and Thomas A. Watson first transmitted sound over wires. - -"This successful experiment was completed in a fifth floor garret -at what was then 109 Court Street and marked the beginning of -world-wide telephone service." - -109 Court Street is long gone. Within sight of Bell's plaque, -across a street, is one of the central offices of NYNEX, -the local Bell RBOC, on 6 Bowdoin Square. - -I cross the street and circle the telco building, slowly, -hands in my jacket pockets. It's a bright, windy, New England -autumn day. The central office is a handsome 1940s-era megalith -in late Art Deco, eight stories high. - -Parked outside the back is a power-generation truck. -The generator strikes me as rather anomalous. Don't they -already have their own generators in this eight-story monster? -Then the suspicion strikes me that NYNEX must have heard -of the September 17 AT&T power-outage which crashed New York City. -Belt-and-suspenders, this generator. Very telco. - -Over the glass doors of the front entrance is a handsome bronze -bas-relief of Art Deco vines, sunflowers, and birds, entwining -the Bell logo and the legend NEW ENGLAND TELEPHONE AND TELEGRAPH COMPANY ---an entity which no longer officially exists. - -The doors are locked securely. I peer through the shadowed glass. -Inside is an official poster reading: - - -"New England Telephone a NYNEX Company - -ATTENTION - -"All persons while on New England Telephone -Company premises are required to visibly wear their -identification cards (C.C.P. Section 2, Page 1). - -"Visitors, vendors, contractors, and all others are -required to visibly wear a daily pass. - -"Thank you. - -Kevin C. Stanton. -Building Security Coordinator." - - -Outside, around the corner, is a pull-down ribbed metal security door, -a locked delivery entrance. Some passing stranger has grafitti-tagged -this door, with a single word in red spray-painted cursive: - -Fury - -# - -My book on the Hacker Crackdown is almost over now. -I have deliberately saved the best for last. - -In February 1991, I attended the CPSR Public Policy Roundtable, -in Washington, DC. CPSR, Computer Professionals for Social Responsibility, -was a sister organization of EFF, or perhaps its aunt, being older -and perhaps somewhat wiser in the ways of the world of politics. - -Computer Professionals for Social Responsibility began in 1981 -in Palo Alto, as an informal discussion group of Californian -computer scientists and technicians, united by nothing more -than an electronic mailing list. This typical high-tech -ad-hocracy received the dignity of its own acronym in 1982, -and was formally incorporated in 1983. - -CPSR lobbied government and public alike with an educational -outreach effort, sternly warning against any foolish -and unthinking trust in complex computer systems. -CPSR insisted that mere computers should never be -considered a magic panacea for humanity's social, -ethical or political problems. CPSR members were especially -troubled about the stability, safety, and dependability -of military computer systems, and very especially troubled -by those systems controlling nuclear arsenals. CPSR was -best-known for its persistent and well-publicized attacks on the -scientific credibility of the Strategic Defense Initiative ("Star Wars"). - -In 1990, CPSR was the nation's veteran cyber-political activist group, -with over two thousand members in twenty- one local chapters across the US. -It was especially active in Boston, Silicon Valley, and Washington DC, -where its Washington office sponsored the Public Policy Roundtable. - -The Roundtable, however, had been funded by EFF, which had passed CPSR -an extensive grant for operations. This was the first large-scale, -official meeting of what was to become the electronic civil -libertarian community. - -Sixty people attended, myself included--in this instance, not so much -as a journalist as a cyberpunk author. Many of the luminaries -of the field took part: Kapor and Godwin as a matter of course. -Richard Civille and Marc Rotenberg of CPSR. Jerry Berman of the ACLU. -John Quarterman, author of The Matrix. Steven Levy, author of Hackers. -George Perry and Sandy Weiss of Prodigy Services, there to network -about the civil-liberties troubles their young commercial -network was experiencing. Dr. Dorothy Denning. Cliff Figallo, -manager of the Well. Steve Jackson was there, having finally -found his ideal target audience, and so was Craig Neidorf, -"Knight Lightning" himself, with his attorney, Sheldon Zenner. -Katie Hafner, science journalist, and co-author of Cyberpunk: -Outlaws and Hackers on the Computer Frontier. Dave Farber, -ARPAnet pioneer and fabled Internet guru. Janlori Goldman -of the ACLU's Project on Privacy and Technology. John Nagle -of Autodesk and the Well. Don Goldberg of the House Judiciary Committee. -Tom Guidoboni, the defense attorney in the Internet Worm case. -Lance Hoffman, computer-science professor at The George Washington -University. Eli Noam of Columbia. And a host of others no less distinguished. - -Senator Patrick Leahy delivered the keynote address, -expressing his determination to keep ahead of the curve -on the issue of electronic free speech. The address was -well-received, and the sense of excitement was palpable. -Every panel discussion was interesting--some were entirely -compelling. People networked with an almost frantic interest. - -I myself had a most interesting and cordial lunch discussion with -Noel and Jeanne Gayler, Admiral Gayler being a former director -of the National Security Agency. As this was the first known encounter -between an actual no-kidding cyberpunk and a chief executive of -America's largest and best-financed electronic espionage apparat, -there was naturally a bit of eyebrow-raising on both sides. - -Unfortunately, our discussion was off-the-record. In fact -all the discussions at the CPSR were officially off-the-record, -the idea being to do some serious networking in an atmosphere -of complete frankness, rather than to stage a media circus. - -In any case, CPSR Roundtable, though interesting and intensely valuable, -was as nothing compared to the truly mind-boggling event that transpired -a mere month later. - -# - -"Computers, Freedom and Privacy." Four hundred people from -every conceivable corner of America's electronic community. -As a science fiction writer, I have been to some weird gigs in my day, -but this thing is truly BEYOND THE PALE. Even "Cyberthon," -Point Foundation's "Woodstock of Cyberspace" where Bay Area -psychedelia collided headlong with the emergent world -of computerized virtual reality, was like a Kiwanis Club gig -compared to this astonishing do. - -The "electronic community" had reached an apogee. -Almost every principal in this book is in attendance. -Civil Libertarians. Computer Cops. The Digital Underground. -Even a few discreet telco people. Colorcoded dots -for lapel tags are distributed. Free Expression issues. -Law Enforcement. Computer Security. Privacy. Journalists. -Lawyers. Educators. Librarians. Programmers. -Stylish punk-black dots for the hackers and phone phreaks. -Almost everyone here seems to wear eight or nine dots, -to have six or seven professional hats. - -It is a community. Something like Lebanon perhaps, -but a digital nation. People who had feuded all year -in the national press, people who entertained the deepest -suspicions of one another's motives and ethics, are now -in each others' laps. "Computers, Freedom and Privacy" -had every reason in the world to turn ugly, and yet except -for small irruptions of puzzling nonsense from the -convention's token lunatic, a surprising bonhomie reigned. -CFP was like a wedding-party in which two lovers, -unstable bride and charlatan groom, tie the knot -in a clearly disastrous matrimony. - -It is clear to both families--even to neighbors and random guests-- -that this is not a workable relationship, and yet the young couple's -desperate attraction can brook no further delay. They simply cannot -help themselves. Crockery will fly, shrieks from their newlywed home -will wake the city block, divorce waits in the wings like a vulture -over the Kalahari, and yet this is a wedding, and there is going -to be a child from it. Tragedies end in death; comedies in marriage. -The Hacker Crackdown is ending in marriage. And there will be a child. - -From the beginning, anomalies reign. John Perry Barlow, -cyberspace ranger, is here. His color photo in -The New York Times Magazine, Barlow scowling -in a grim Wyoming snowscape, with long black coat, -dark hat, a Macintosh SE30 propped on a fencepost -and an awesome frontier rifle tucked under one arm, -will be the single most striking visual image -of the Hacker Crackdown. And he is CFP's guest of honor-- -along with Gail Thackeray of the FCIC! What on earth do -they expect these dual guests to do with each other? Waltz? - -Barlow delivers the first address. Uncharacteristically, -he is hoarse--the sheer volume of roadwork has worn him down. -He speaks briefly, congenially, in a plea for conciliation, -and takes his leave to a storm of applause. - -Then Gail Thackeray takes the stage. She's visibly nervous. -She's been on the Well a lot lately. Reading those Barlow posts. -Following Barlow is a challenge to anyone. In honor of the famous -lyricist for the Grateful Dead, she announces reedily, she is going to read-- -A POEM. A poem she has composed herself. - -It's an awful poem, doggerel in the rollicking meter of Robert W. Service's -The Cremation of Sam McGee, but it is in fact, a poem. It's the Ballad -of the Electronic Frontier! A poem about the Hacker Crackdown and the -sheer unlikelihood of CFP. It's full of in-jokes. The score or so cops -in the audience, who are sitting together in a nervous claque, -are absolutely cracking-up. Gail's poem is the funniest goddamn thing -they've ever heard. The hackers and civil-libs, who had this woman figured -for Ilsa She-Wolf of the SS, are staring with their jaws hanging loosely. -Never in the wildest reaches of their imagination had they figured -Gail Thackeray was capable of such a totally off-the-wall move. -You can see them punching their mental CONTROL-RESET buttons. -Jesus! This woman's a hacker weirdo! She's JUST LIKE US! -God, this changes everything! - -Al Bayse, computer technician for the FBI, had been the only cop -at the CPSR Roundtable, dragged there with his arm bent by -Dorothy Denning. He was guarded and tightlipped at CPSR Roundtable; -a "lion thrown to the Christians." - -At CFP, backed by a claque of cops, Bayse suddenly waxes eloquent -and even droll, describing the FBI's "NCIC 2000", a gigantic digital catalog -of criminal records, as if he has suddenly become some weird hybrid -of George Orwell and George Gobel. Tentatively, he makes an arcane -joke about statistical analysis. At least a third of the crowd laughs aloud. - -"They didn't laugh at that at my last speech," Bayse observes. -He had been addressing cops--STRAIGHT cops, not computer people. -It had been a worthy meeting, useful one supposes, but nothing like THIS. -There has never been ANYTHING like this. Without any prodding, -without any preparation, people in the audience simply begin to ask questions. -Longhairs, freaky people, mathematicians. Bayse is answering, politely, -frankly, fully, like a man walking on air. The ballroom's atmosphere -crackles with surreality. A female lawyer behind me breaks into a sweat -and a hot waft of surprisingly potent and musky perfume flows off -her pulse-points. - -People are giddy with laughter. People are interested, -fascinated, their eyes so wide and dark that they seem eroticized. -Unlikely daisy-chains form in the halls, around the bar, on the escalators: -cops with hackers, civil rights with FBI, Secret Service with phone phreaks. - -Gail Thackeray is at her crispest in a white wool sweater with a -tiny Secret Service logo. "I found Phiber Optik at the payphones, -and when he saw my sweater, he turned into a PILLAR OF SALT!" she chortles. - -Phiber discusses his case at much length with his arresting officer, -Don Delaney of the New York State Police. After an hour's chat, -the two of them look ready to begin singing "Auld Lang Syne." -Phiber finally finds the courage to get his worst complaint off his chest. -It isn't so much the arrest. It was the CHARGE. Pirating service -off 900 numbers. I'm a PROGRAMMER, Phiber insists. This lame charge -is going to hurt my reputation. It would have been cool to be busted -for something happening, like Section 1030 computer intrusion. -Maybe some kind of crime that's scarcely been invented yet. -Not lousy phone fraud. Phooey. - -Delaney seems regretful. He had a mountain of possible criminal charges -against Phiber Optik. The kid's gonna plead guilty anyway. He's a -first timer, they always plead. Coulda charged the kid with most anything, -and gotten the same result in the end. Delaney seems genuinely sorry -not to have gratified Phiber in this harmless fashion. Too late now. -Phiber's pled already. All water under the bridge. Whaddya gonna do? - -Delaney's got a good grasp on the hacker mentality. -He held a press conference after he busted a bunch of -Masters of Deception kids. Some journo had asked him: -"Would you describe these people as GENIUSES?" -Delaney's deadpan answer, perfect: "No, I would describe -these people as DEFENDANTS." Delaney busts a kid for -hacking codes with repeated random dialling. Tells the -press that NYNEX can track this stuff in no time flat nowadays, -and a kid has to be STUPID to do something so easy to catch. -Dead on again: hackers don't mind being thought of as Genghis Khan -by the straights, but if there's anything that really gets 'em -where they live, it's being called DUMB. - -Won't be as much fun for Phiber next time around. -As a second offender he's gonna see prison. -Hackers break the law. They're not geniuses, either. -They're gonna be defendants. And yet, Delaney muses over -a drink in the hotel bar, he has found it impossible to treat -them as common criminals. Delaney knows criminals. These kids, -by comparison, are clueless--there is just no crook vibe off of them, -they don't smell right, they're just not BAD. - -Delaney has seen a lot of action. He did Vietnam. -He's been shot at, he has shot people. He's a homicide -cop from New York. He has the appearance of a man who -has not only seen the shit hit the fan but has seen it splattered -across whole city blocks and left to ferment for years. -This guy has been around. - -He listens to Steve Jackson tell his story. The dreamy -game strategist has been dealt a bad hand. He has played -it for all he is worth. Under his nerdish SF-fan exterior -is a core of iron. Friends of his say Steve Jackson believes -in the rules, believes in fair play. He will never compromise -his principles, never give up. "Steve," Delaney says to -Steve Jackson, "they had some balls, whoever busted you. -You're all right!" Jackson, stunned, falls silent and -actually blushes with pleasure. - -Neidorf has grown up a lot in the past year. The kid is -a quick study, you gotta give him that. Dressed by his mom, -the fashion manager for a national clothing chain, -Missouri college techie-frat Craig Neidorf out-dappers -everyone at this gig but the toniest East Coast lawyers. -The iron jaws of prison clanged shut without him and now -law school beckons for Neidorf. He looks like a larval Congressman. - -Not a "hacker," our Mr. Neidorf. He's not interested -in computer science. Why should he be? He's not -interested in writing C code the rest of his life, -and besides, he's seen where the chips fall. -To the world of computer science he and Phrack -were just a curiosity. But to the world of law. . . . -The kid has learned where the bodies are buried. -He carries his notebook of press clippings wherever he goes. - -Phiber Optik makes fun of Neidorf for a Midwestern geek, -for believing that "Acid Phreak" does acid and listens to acid rock. -Hell no. Acid's never done ACID! Acid's into ACID HOUSE MUSIC. -Jesus. The very idea of doing LSD. Our PARENTS did LSD, ya clown. - -Thackeray suddenly turns upon Craig Neidorf the full lighthouse -glare of her attention and begins a determined half-hour attempt -to WIN THE BOY OVER. The Joan of Arc of Computer Crime is -GIVING CAREER ADVICE TO KNIGHT LIGHTNING! "Your experience -would be very valuable--a real asset," she tells him with -unmistakeable sixty-thousand-watt sincerity. Neidorf is fascinated. -He listens with unfeigned attention. He's nodding and saying yes ma'am. -Yes, Craig, you too can forget all about money and enter the glamorous -and horribly underpaid world of PROSECUTING COMPUTER CRIME! -You can put your former friends in prison--ooops. . . . - -You cannot go on dueling at modem's length indefinitely. -You cannot beat one another senseless with rolled-up press-clippings. -Sooner or later you have to come directly to grips. -And yet the very act of assembling here has changed -the entire situation drastically. John Quarterman, -author of The Matrix, explains the Internet at his symposium. -It is the largest news network in the world, it is growing -by leaps and bounds, and yet you cannot measure Internet because -you cannot stop it in place. It cannot stop, because there -is no one anywhere in the world with the authority to stop Internet. -It changes, yes, it grows, it embeds itself across the post-industrial, -postmodern world and it generates community wherever it -touches, and it is doing this all by itself. - -Phiber is different. A very fin de siecle kid, Phiber Optik. -Barlow says he looks like an Edwardian dandy. He does rather. -Shaven neck, the sides of his skull cropped hip-hop close, -unruly tangle of black hair on top that looks pomaded, -he stays up till four a.m. and misses all the sessions, -then hangs out in payphone booths with his acoustic coupler -gutsily CRACKING SYSTEMS RIGHT IN THE MIDST OF THE HEAVIEST -LAW ENFORCEMENT DUDES IN THE U.S., or at least PRETENDING to. . . . -Unlike "Frank Drake." Drake, who wrote Dorothy Denning out -of nowhere, and asked for an interview for his cheapo -cyberpunk fanzine, and then started grilling her on her ethics. -She was squirmin', too. . . . Drake, scarecrow-tall with his -floppy blond mohawk, rotting tennis shoes and black leather jacket -lettered ILLUMINATI in red, gives off an unmistakeable air -of the bohemian literatus. Drake is the kind of guy -who reads British industrial design magazines and appreciates -William Gibson because the quality of the prose is so tasty. -Drake could never touch a phone or a keyboard again, -and he'd still have the nose-ring and the blurry photocopied -fanzines and the sampled industrial music. He's a radical punk -with a desktop-publishing rig and an Internet address. -Standing next to Drake, the diminutive Phiber looks like he's -been physically coagulated out of phone-lines. Born to phreak. - -Dorothy Denning approaches Phiber suddenly. The two of them -are about the same height and body-build. Denning's blue eyes -flash behind the round window-frames of her glasses. -"Why did you say I was `quaint?'" she asks Phiber, quaintly. - -It's a perfect description but Phiber is nonplussed. . . -"Well, I uh, you know. . . ." - -"I also think you're quaint, Dorothy," I say, novelist to the rescue, -the journo gift of gab. . . . She is neat and dapper and yet there's -an arcane quality to her, something like a Pilgrim Maiden behind -leaded glass; if she were six inches high Dorothy Denning would look -great inside a china cabinet. . .The Cryptographeress. . . -The Cryptographrix. . .whatever. . . . Weirdly, Peter Denning looks -just like his wife, you could pick this gentleman out of a thousand guys -as the soulmate of Dorothy Denning. Wearing tailored slacks, -a spotless fuzzy varsity sweater, and a neatly knotted academician's tie. . . . -This fineboned, exquisitely polite, utterly civilized and hyperintelligent -couple seem to have emerged from some cleaner and finer parallel universe, -where humanity exists to do the Brain Teasers column in Scientific American. -Why does this Nice Lady hang out with these unsavory characters? - -Because the time has come for it, that's why. -Because she's the best there is at what she does. - -Donn Parker is here, the Great Bald Eagle of Computer Crime. . . . -With his bald dome, great height, and enormous Lincoln-like hands, -the great visionary pioneer of the field plows through the lesser mortals -like an icebreaker. . . . His eyes are fixed on the future with the -rigidity of a bronze statue. . . . Eventually, he tells his audience, -all business crime will be computer crime, because businesses will do -everything through computers. "Computer crime" as a category will vanish. - -In the meantime, passing fads will flourish and fail and evaporate. . . . -Parker's commanding, resonant voice is sphinxlike, everything is viewed -from some eldritch valley of deep historical abstraction. . . . -Yes, they've come and they've gone, these passing flaps in the world -of digital computation. . . . The radio-frequency emanation scandal. . . -KGB and MI5 and CIA do it every day, it's easy, but nobody else ever has. . . . -The salami-slice fraud, mostly mythical. . . . "Crimoids," he calls them. . . . -Computer viruses are the current crimoid champ, a lot less dangerous than -most people let on, but the novelty is fading and there's a crimoid vacuum at -the moment, the press is visibly hungering for something more outrageous. . . . -The Great Man shares with us a few speculations on the coming crimoids. . . . -Desktop Forgery! Wow. . . . Computers stolen just for the sake of the -information within them--data-napping! Happened in Britain a while ago, -could be the coming thing. . . . Phantom nodes in the Internet! - -Parker handles his overhead projector sheets with an ecclesiastical air. . . . -He wears a grey double-breasted suit, a light blue shirt, and a -very quiet tie of understated maroon and blue paisley. . . . -Aphorisms emerge from him with slow, leaden emphasis. . . . -There is no such thing as an adequately secure computer -when one faces a sufficiently powerful adversary. . . . -Deterrence is the most socially useful aspect of security. . . . -People are the primary weakness in all information systems. . . . -The entire baseline of computer security must be shifted upward. . . . -Don't ever violate your security by publicly describing -your security measures. . . . - -People in the audience are beginning to squirm, and yet -there is something about the elemental purity of this guy's -philosophy that compels uneasy respect. . . . Parker sounds -like the only sane guy left in the lifeboat, sometimes. -The guy who can prove rigorously, from deep moral principles, -that Harvey there, the one with the broken leg and the checkered past, -is the one who has to be, err. . .that is, Mr. Harvey is best placed -to make the necessary sacrifice for the security and indeed -the very survival of the rest of this lifeboat's crew. . . . -Computer security, Parker informs us mournfully, is a -nasty topic, and we wish we didn't have to have it. . . . -The security expert, armed with method and logic, must think--imagine-- -everything that the adversary might do before the adversary might -actually do it. It is as if the criminal's dark brain were an -extensive subprogram within the shining cranium of Donn Parker. -He is a Holmes whose Moriarty does not quite yet exist -and so must be perfectly simulated. - -CFP is a stellar gathering, with the giddiness of a wedding. -It is a happy time, a happy ending, they know their world -is changing forever tonight, and they're proud to have been there -to see it happen, to talk, to think, to help. - -And yet as night falls, a certain elegiac quality manifests itself, -as the crowd gathers beneath the chandeliers with their wineglasses -and dessert plates. Something is ending here, gone forever, -and it takes a while to pinpoint it. - -It is the End of the Amateurs. - - - - - - - - - -End of the Project Gutenberg EBook of Hacker Crackdown, by Bruce Sterling - -*** END OF THIS PROJECT GUTENBERG EBOOK HACKER CRACKDOWN *** - -***** This file should be named 101.txt or 101.zip ***** -This and all associated files of various formats will be found in: - http://www.gutenberg.org/1/0/101/ - - - -Updated editions will replace the previous one--the old editions will be -renamed. - -Creating the works from public domain print editions means that no one -owns a United States copyright in these works, so the Foundation (and -you!) can copy and distribute it in the United States without permission -and without paying copyright royalties. Special rules, set forth in the -General Terms of Use part of this license, apply to copying and -distributing Project Gutenberg-tm electronic works to protect the -PROJECT GUTENBERG-tm concept and trademark. Project Gutenberg is a -registered trademark, and may not be used if you charge for the eBooks, -unless you receive specific permission. If you do not charge anything -for copies of this eBook, complying with the rules is very easy. You may -use this eBook for nearly any purpose such as creation of derivative -works, reports, performances and research. They may be modified and -printed and given away--you may do practically ANYTHING with public -domain eBooks. Redistribution is subject to the trademark license, -especially commercial redistribution. - - - -*** START: FULL LICENSE *** - -THE FULL PROJECT GUTENBERG LICENSE -PLEASE READ THIS BEFORE YOU DISTRIBUTE OR USE THIS WORK - -To protect the Project Gutenberg-tm mission of promoting the free -distribution of electronic works, by using or distributing this work -(or any other work associated in any way with the phrase "Project -Gutenberg"), you agree to comply with all the terms of the Full Project -Gutenberg-tm License (available with this file or online at -http://www.gutenberg.org/license). - - -Section 1. General Terms of Use and Redistributing Project Gutenberg-tm -electronic works - -1.A. By reading or using any part of this Project Gutenberg-tm -electronic work, you indicate that you have read, understand, agree to -and accept all the terms of this license and intellectual property -(trademark/copyright) agreement. If you do not agree to abide by all -the terms of this agreement, you must cease using and return or destroy -all copies of Project Gutenberg-tm electronic works in your possession. -If you paid a fee for obtaining a copy of or access to a Project -Gutenberg-tm electronic work and you do not agree to be bound by the -terms of this agreement, you may obtain a refund from the person or -entity to whom you paid the fee as set forth in paragraph 1.E.8. - -1.B. "Project Gutenberg" is a registered trademark. It may only be -used on or associated in any way with an electronic work by people who -agree to be bound by the terms of this agreement. There are a few -things that you can do with most Project Gutenberg-tm electronic works -even without complying with the full terms of this agreement. See -paragraph 1.C below. There are a lot of things you can do with Project -Gutenberg-tm electronic works if you follow the terms of this agreement -and help preserve free future access to Project Gutenberg-tm electronic -works. See paragraph 1.E below. - -1.C. The Project Gutenberg Literary Archive Foundation ("the Foundation" -or PGLAF), owns a compilation copyright in the collection of Project -Gutenberg-tm electronic works. Nearly all the individual works in the -collection are in the public domain in the United States. If an -individual work is in the public domain in the United States and you are -located in the United States, we do not claim a right to prevent you from -copying, distributing, performing, displaying or creating derivative -works based on the work as long as all references to Project Gutenberg -are removed. Of course, we hope that you will support the Project -Gutenberg-tm mission of promoting free access to electronic works by -freely sharing Project Gutenberg-tm works in compliance with the terms of -this agreement for keeping the Project Gutenberg-tm name associated with -the work. You can easily comply with the terms of this agreement by -keeping this work in the same format with its attached full Project -Gutenberg-tm License when you share it without charge with others. -This particular work is one of the few copyrighted individual works -included with the permission of the copyright holder. Information on -the copyright owner for this particular work and the terms of use -imposed by the copyright holder on this work are set forth at the -beginning of this work. - -1.D. The copyright laws of the place where you are located also govern -what you can do with this work. Copyright laws in most countries are in -a constant state of change. If you are outside the United States, check -the laws of your country in addition to the terms of this agreement -before downloading, copying, displaying, performing, distributing or -creating derivative works based on this work or any other Project -Gutenberg-tm work. The Foundation makes no representations concerning -the copyright status of any work in any country outside the United -States. - -1.E. Unless you have removed all references to Project Gutenberg: - -1.E.1. The following sentence, with active links to, or other immediate -access to, the full Project Gutenberg-tm License must appear prominently -whenever any copy of a Project Gutenberg-tm work (any work on which the -phrase "Project Gutenberg" appears, or with which the phrase "Project -Gutenberg" is associated) is accessed, displayed, performed, viewed, -copied or distributed: - -This eBook is for the use of anyone anywhere at no cost and with -almost no restrictions whatsoever. You may copy it, give it away or -re-use it under the terms of the Project Gutenberg License included -with this eBook or online at www.gutenberg.org - -1.E.2. If an individual Project Gutenberg-tm electronic work is derived -from the public domain (does not contain a notice indicating that it is -posted with permission of the copyright holder), the work can be copied -and distributed to anyone in the United States without paying any fees -or charges. If you are redistributing or providing access to a work -with the phrase "Project Gutenberg" associated with or appearing on the -work, you must comply either with the requirements of paragraphs 1.E.1 -through 1.E.7 or obtain permission for the use of the work and the -Project Gutenberg-tm trademark as set forth in paragraphs 1.E.8 or -1.E.9. - -1.E.3. If an individual Project Gutenberg-tm electronic work is posted -with the permission of the copyright holder, your use and distribution -must comply with both paragraphs 1.E.1 through 1.E.7 and any additional -terms imposed by the copyright holder. Additional terms will be linked -to the Project Gutenberg-tm License for all works posted with the -permission of the copyright holder found at the beginning of this work. - -1.E.4. Do not unlink or detach or remove the full Project Gutenberg-tm -License terms from this work, or any files containing a part of this -work or any other work associated with Project Gutenberg-tm. - -1.E.5. Do not copy, display, perform, distribute or redistribute this -electronic work, or any part of this electronic work, without -prominently displaying the sentence set forth in paragraph 1.E.1 with -active links or immediate access to the full terms of the Project -Gutenberg-tm License. - -1.E.6. You may convert to and distribute this work in any binary, -compressed, marked up, nonproprietary or proprietary form, including any -word processing or hypertext form. However, if you provide access to or -distribute copies of a Project Gutenberg-tm work in a format other than -"Plain Vanilla ASCII" or other format used in the official version -posted on the official Project Gutenberg-tm web site (www.gutenberg.org), -you must, at no additional cost, fee or expense to the user, provide a -copy, a means of exporting a copy, or a means of obtaining a copy upon -request, of the work in its original "Plain Vanilla ASCII" or other -form. Any alternate format must include the full Project Gutenberg-tm -License as specified in paragraph 1.E.1. - -1.E.7. Do not charge a fee for access to, viewing, displaying, -performing, copying or distributing any Project Gutenberg-tm works -unless you comply with paragraph 1.E.8 or 1.E.9. - -1.E.8. You may charge a reasonable fee for copies of or providing -access to or distributing Project Gutenberg-tm electronic works provided -that - -- You pay a royalty fee of 20% of the gross profits you derive from - the use of Project Gutenberg-tm works calculated using the method - you already use to calculate your applicable taxes. The fee is - owed to the owner of the Project Gutenberg-tm trademark, but he - has agreed to donate royalties under this paragraph to the - Project Gutenberg Literary Archive Foundation. Royalty payments - must be paid within 60 days following each date on which you - prepare (or are legally required to prepare) your periodic tax - returns. Royalty payments should be clearly marked as such and - sent to the Project Gutenberg Literary Archive Foundation at the - address specified in Section 4, "Information about donations to - the Project Gutenberg Literary Archive Foundation." - -- You provide a full refund of any money paid by a user who notifies - you in writing (or by e-mail) within 30 days of receipt that s/he - does not agree to the terms of the full Project Gutenberg-tm - License. You must require such a user to return or - destroy all copies of the works possessed in a physical medium - and discontinue all use of and all access to other copies of - Project Gutenberg-tm works. - -- You provide, in accordance with paragraph 1.F.3, a full refund of any - money paid for a work or a replacement copy, if a defect in the - electronic work is discovered and reported to you within 90 days - of receipt of the work. - -- You comply with all other terms of this agreement for free - distribution of Project Gutenberg-tm works. - -1.E.9. If you wish to charge a fee or distribute a Project Gutenberg-tm -electronic work or group of works on different terms than are set -forth in this agreement, you must obtain permission in writing from -both the Project Gutenberg Literary Archive Foundation and Michael -Hart, the owner of the Project Gutenberg-tm trademark. Contact the -Foundation as set forth in Section 3 below. - -1.F. - -1.F.1. Project Gutenberg volunteers and employees expend considerable -effort to identify, do copyright research on, transcribe and proofread -public domain works in creating the Project Gutenberg-tm -collection. Despite these efforts, Project Gutenberg-tm electronic -works, and the medium on which they may be stored, may contain -"Defects," such as, but not limited to, incomplete, inaccurate or -corrupt data, transcription errors, a copyright or other intellectual -property infringement, a defective or damaged disk or other medium, a -computer virus, or computer codes that damage or cannot be read by -your equipment. - -1.F.2. LIMITED WARRANTY, DISCLAIMER OF DAMAGES - Except for the "Right -of Replacement or Refund" described in paragraph 1.F.3, the Project -Gutenberg Literary Archive Foundation, the owner of the Project -Gutenberg-tm trademark, and any other party distributing a Project -Gutenberg-tm electronic work under this agreement, disclaim all -liability to you for damages, costs and expenses, including legal -fees. YOU AGREE THAT YOU HAVE NO REMEDIES FOR NEGLIGENCE, STRICT -LIABILITY, BREACH OF WARRANTY OR BREACH OF CONTRACT EXCEPT THOSE -PROVIDED IN PARAGRAPH 1.F.3. YOU AGREE THAT THE FOUNDATION, THE -TRADEMARK OWNER, AND ANY DISTRIBUTOR UNDER THIS AGREEMENT WILL NOT BE -LIABLE TO YOU FOR ACTUAL, DIRECT, INDIRECT, CONSEQUENTIAL, PUNITIVE OR -INCIDENTAL DAMAGES EVEN IF YOU GIVE NOTICE OF THE POSSIBILITY OF SUCH -DAMAGE. - -1.F.3. LIMITED RIGHT OF REPLACEMENT OR REFUND - If you discover a -defect in this electronic work within 90 days of receiving it, you can -receive a refund of the money (if any) you paid for it by sending a -written explanation to the person you received the work from. If you -received the work on a physical medium, you must return the medium with -your written explanation. The person or entity that provided you with -the defective work may elect to provide a replacement copy in lieu of a -refund. If you received the work electronically, the person or entity -providing it to you may choose to give you a second opportunity to -receive the work electronically in lieu of a refund. If the second copy -is also defective, you may demand a refund in writing without further -opportunities to fix the problem. - -1.F.4. Except for the limited right of replacement or refund set forth -in paragraph 1.F.3, this work is provided to you 'AS-IS,' WITH NO OTHER -WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -WARRANTIES OF MERCHANTIBILITY OR FITNESS FOR ANY PURPOSE. - -1.F.5. Some states do not allow disclaimers of certain implied -warranties or the exclusion or limitation of certain types of damages. -If any disclaimer or limitation set forth in this agreement violates the -law of the state applicable to this agreement, the agreement shall be -interpreted to make the maximum disclaimer or limitation permitted by -the applicable state law. The invalidity or unenforceability of any -provision of this agreement shall not void the remaining provisions. - -1.F.6. INDEMNITY - You agree to indemnify and hold the Foundation, the -trademark owner, any agent or employee of the Foundation, anyone -providing copies of Project Gutenberg-tm electronic works in accordance -with this agreement, and any volunteers associated with the production, -promotion and distribution of Project Gutenberg-tm electronic works, -harmless from all liability, costs and expenses, including legal fees, -that arise directly or indirectly from any of the following which you do -or cause to occur: (a) distribution of this or any Project Gutenberg-tm -work, (b) alteration, modification, or additions or deletions to any -Project Gutenberg-tm work, and (c) any Defect you cause. - - -Section 2. Information about the Mission of Project Gutenberg-tm - -Project Gutenberg-tm is synonymous with the free distribution of -electronic works in formats readable by the widest variety of computers -including obsolete, old, middle-aged and new computers. It exists -because of the efforts of hundreds of volunteers and donations from -people in all walks of life. - -Volunteers and financial support to provide volunteers with the -assistance they need are critical to reaching Project Gutenberg-tm's -goals and ensuring that the Project Gutenberg-tm collection will -remain freely available for generations to come. In 2001, the Project -Gutenberg Literary Archive Foundation was created to provide a secure -and permanent future for Project Gutenberg-tm and future generations. -To learn more about the Project Gutenberg Literary Archive Foundation -and how your efforts and donations can help, see Sections 3 and 4 -and the Foundation web page at http://www.pglaf.org. - - -Section 3. Information about the Project Gutenberg Literary Archive -Foundation - -The Project Gutenberg Literary Archive Foundation is a non profit -501(c)(3) educational corporation organized under the laws of the -state of Mississippi and granted tax exempt status by the Internal -Revenue Service. The Foundation's EIN or federal tax identification -number is 64-6221541. Its 501(c)(3) letter is posted at -http://pglaf.org/fundraising. Contributions to the Project Gutenberg -Literary Archive Foundation are tax deductible to the full extent -permitted by U.S. federal laws and your state's laws. - -The Foundation's principal office is located at 4557 Melan Dr. S. -Fairbanks, AK, 99712., but its volunteers and employees are scattered -throughout numerous locations. Its business office is located at -809 North 1500 West, Salt Lake City, UT 84116, (801) 596-1887, email -business@pglaf.org. Email contact links and up to date contact -information can be found at the Foundation's web site and official -page at http://pglaf.org - -For additional contact information: - Dr. Gregory B. Newby - Chief Executive and Director - gbnewby@pglaf.org - -Section 4. Information about Donations to the Project Gutenberg -Literary Archive Foundation - -Project Gutenberg-tm depends upon and cannot survive without wide -spread public support and donations to carry out its mission of -increasing the number of public domain and licensed works that can be -freely distributed in machine readable form accessible by the widest -array of equipment including outdated equipment. Many small donations -($1 to $5,000) are particularly important to maintaining tax exempt -status with the IRS. - -The Foundation is committed to complying with the laws regulating -charities and charitable donations in all 50 states of the United -States. Compliance requirements are not uniform and it takes a -considerable effort, much paperwork and many fees to meet and keep up -with these requirements. We do not solicit donations in locations -where we have not received written confirmation of compliance. To -SEND DONATIONS or determine the status of compliance for any -particular state visit http://pglaf.org - -While we cannot and do not solicit contributions from states where we -have not met the solicitation requirements, we know of no prohibition -against accepting unsolicited donations from donors in such states who -approach us with offers to donate. - -International donations are gratefully accepted, but we cannot make -any statements concerning tax treatment of donations received from -outside the United States. U.S. laws alone swamp our small staff. - -Please check the Project Gutenberg Web pages for current donation -methods and addresses. Donations are accepted in a number of other -ways including checks, online payments and credit card donations. -To donate, please visit: http://pglaf.org/donate - - -Section 5. General Information About Project Gutenberg-tm electronic -works. - -Professor Michael S. Hart is the originator of the Project Gutenberg-tm -concept of a library of electronic works that could be freely shared -with anyone. For thirty years, he produced and distributed Project -Gutenberg-tm eBooks with only a loose network of volunteer support. - -Project Gutenberg-tm eBooks are often created from several printed -editions, all of which are confirmed as Public Domain in the U.S. -unless a copyright notice is included. Thus, we do not necessarily -keep eBooks in compliance with any particular paper edition. - -Each eBook is in a subdirectory of the same number as the eBook's -eBook number, often in several formats including plain vanilla ASCII, -compressed (zipped), HTML and others. - -Corrected EDITIONS of our eBooks replace the old file and take over -the old filename and etext number. The replaced older file is renamed. -VERSIONS based on separate sources are treated as new eBooks receiving -new filenames and etext numbers. - -Most people start at our Web site which has the main PG search facility: - -http://www.gutenberg.org - -This Web site includes information about Project Gutenberg-tm, -including how to make donations to the Project Gutenberg Literary -Archive Foundation, how to help produce our new eBooks, and how to -subscribe to our email newsletter to hear about new eBooks. - -EBooks posted prior to November 2003, with eBook numbers BELOW #10000, -are filed in directories based on their release date. If you want to -download any of these eBooks directly, rather than using the regular -search system you may utilize the following addresses and just -download by the etext year. - -http://www.ibiblio.org/gutenberg/etext06 - - (Or /etext 05, 04, 03, 02, 01, 00, 99, - 98, 97, 96, 95, 94, 93, 92, 92, 91 or 90) - -EBooks posted since November 2003, with etext numbers OVER #10000, are -filed in a different way. The year of a release date is no longer part -of the directory path. The path is based on the etext number (which is -identical to the filename). The path to the file is made up of single -digits corresponding to all but the last digit in the filename. For -example an eBook of filename 10234 would be found at: - -http://www.gutenberg.org/1/0/2/3/10234 - -or filename 24689 would be found at: -http://www.gutenberg.org/2/4/6/8/24689 - -An alternative method of locating eBooks: -http://www.gutenberg.org/GUTINDEX.ALL - -*** END: FULL LICENSE *** diff --git a/common/src/leap/soledad/common/tests/server_state.py b/common/src/leap/soledad/common/tests/server_state.py deleted file mode 100644 index 26838f89..00000000 --- a/common/src/leap/soledad/common/tests/server_state.py +++ /dev/null @@ -1,81 +0,0 @@ -# -*- coding: utf-8 -*- -# server_state.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -State for servers to be used in tests. -""" - - -import os -import errno -import tempfile - - -from leap.soledad.common.l2db.remote.server_state import ServerState -from leap.soledad.common.tests.util import ( - copy_sqlcipher_database_for_test, -) - - -class ServerStateForTests(ServerState): - - """Passed to a Request when it is instantiated. - - This is used to track server-side state, such as working-directory, open - databases, etc. - """ - - def __init__(self): - self._workingdir = tempfile.mkdtemp() - - def _relpath(self, relpath): - return os.path.join(self._workingdir, relpath) - - def open_database(self, path): - """Open a database at the given location.""" - from leap.soledad.client.sqlcipher import SQLCipherDatabase - return SQLCipherDatabase.open_database(path, '123', False) - - def create_database(self, path): - """Create a database at the given location.""" - from leap.soledad.client.sqlcipher import SQLCipherDatabase - return SQLCipherDatabase.open_database(path, '123', True) - - def check_database(self, path): - """Check if the database at the given location exists. - - Simply returns if it does or raises DatabaseDoesNotExist. - """ - db = self.open_database(path) - db.close() - - def ensure_database(self, path): - """Ensure database at the given location.""" - from leap.soledad.client.sqlcipher import SQLCipherDatabase - full_path = self._relpath(path) - db = SQLCipherDatabase.open_database(full_path, '123', False) - return db, db._replica_uid - - def delete_database(self, path): - """Delete database at the given location.""" - from leap.u1db.backends import sqlite_backend - full_path = self._relpath(path) - sqlite_backend.SQLiteDatabase.delete_database(full_path) - - def _copy_database(self, db): - return copy_sqlcipher_database_for_test(None, db) diff --git a/common/src/leap/soledad/common/tests/test_async.py b/common/src/leap/soledad/common/tests/test_async.py deleted file mode 100644 index 52be4ff3..00000000 --- a/common/src/leap/soledad/common/tests/test_async.py +++ /dev/null @@ -1,142 +0,0 @@ -# -*- coding: utf-8 -*- -# test_async.py -# Copyright (C) 2013, 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -import os -import hashlib - -from twisted.internet import defer - -from leap.soledad.common.tests.util import BaseSoledadTest -from leap.soledad.client import adbapi -from leap.soledad.client.sqlcipher import SQLCipherOptions - - -class ASyncSQLCipherRetryTestCase(BaseSoledadTest): - - """ - Test asynchronous SQLCipher operation. - """ - - NUM_DOCS = 5000 - - def _get_dbpool(self): - tmpdb = os.path.join(self.tempdir, "test.soledad") - opts = SQLCipherOptions(tmpdb, "secret", create=True) - return adbapi.getConnectionPool(opts) - - def _get_sample(self): - if not getattr(self, "_sample", None): - dirname = os.path.dirname(os.path.realpath(__file__)) - sample_file = os.path.join(dirname, "hacker_crackdown.txt") - with open(sample_file) as f: - self._sample = f.readlines() - return self._sample - - def test_concurrent_puts_fail_with_few_retries_and_small_timeout(self): - """ - Test if concurrent updates to the database with small timeout and - small number of retries fail with "database is locked" error. - - Many concurrent write attempts to the same sqlcipher database may fail - when the timeout is small and there are no retries. This test will - pass if any of the attempts to write the database fail. - - This test is much dependent on the environment and its result intends - to contrast with the test for the workaround for the "database is - locked" problem, which is addressed by the "test_concurrent_puts" test - below. - - If this test ever fails, it means that either (1) the platform where - you are running is it very powerful and you should try with an even - lower timeout value, or (2) the bug has been solved by a better - implementation of the underlying database pool, and thus this test - should be removed from the test suite. - """ - - old_timeout = adbapi.SQLCIPHER_CONNECTION_TIMEOUT - old_max_retries = adbapi.SQLCIPHER_MAX_RETRIES - - adbapi.SQLCIPHER_CONNECTION_TIMEOUT = 1 - adbapi.SQLCIPHER_MAX_RETRIES = 1 - - dbpool = self._get_dbpool() - - def _create_doc(doc): - return dbpool.runU1DBQuery("create_doc", doc) - - def _insert_docs(): - deferreds = [] - for i in range(self.NUM_DOCS): - payload = self._get_sample()[i] - chash = hashlib.sha256(payload).hexdigest() - doc = {"number": i, "payload": payload, 'chash': chash} - d = _create_doc(doc) - deferreds.append(d) - return defer.gatherResults(deferreds, consumeErrors=True) - - def _errback(e): - if e.value[0].getErrorMessage() == "database is locked": - adbapi.SQLCIPHER_CONNECTION_TIMEOUT = old_timeout - adbapi.SQLCIPHER_MAX_RETRIES = old_max_retries - return defer.succeed("") - raise Exception - - d = _insert_docs() - d.addCallback(lambda _: dbpool.runU1DBQuery("get_all_docs")) - d.addErrback(_errback) - return d - - def test_concurrent_puts(self): - """ - Test that many concurrent puts succeed. - - Currently, there's a known problem with the concurrent database pool - which is that many concurrent attempts to write to the database may - fail when the lock timeout is small and when there are no (or few) - retries. We currently workaround this problem by increasing the - timeout and the number of retries. - - Should this test ever fail, it probably means that the timeout and/or - number of retries should be increased for the platform you're running - the test. If the underlying database pool is ever fixed, then the test - above will fail and we should remove this comment from here. - """ - - dbpool = self._get_dbpool() - - def _create_doc(doc): - return dbpool.runU1DBQuery("create_doc", doc) - - def _insert_docs(): - deferreds = [] - for i in range(self.NUM_DOCS): - payload = self._get_sample()[i] - chash = hashlib.sha256(payload).hexdigest() - doc = {"number": i, "payload": payload, 'chash': chash} - d = _create_doc(doc) - deferreds.append(d) - return defer.gatherResults(deferreds, consumeErrors=True) - - def _count_docs(results): - _, docs = results - if self.NUM_DOCS == len(docs): - return defer.succeed("") - raise Exception - - d = _insert_docs() - d.addCallback(lambda _: dbpool.runU1DBQuery("get_all_docs")) - d.addCallback(_count_docs) - return d diff --git a/common/src/leap/soledad/common/tests/test_couch.py b/common/src/leap/soledad/common/tests/test_couch.py deleted file mode 100644 index eefefc5d..00000000 --- a/common/src/leap/soledad/common/tests/test_couch.py +++ /dev/null @@ -1,1442 +0,0 @@ -# -*- coding: utf-8 -*- -# test_couch.py -# Copyright (C) 2013-2016 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Test ObjectStore and Couch backend bits. -""" -import json - -from uuid import uuid4 -from urlparse import urljoin - -from couchdb.client import Server - -from testscenarios import TestWithScenarios -from twisted.trial import unittest -from mock import Mock - -from leap.soledad.common.l2db import errors as u1db_errors -from leap.soledad.common.l2db import SyncTarget -from leap.soledad.common.l2db import vectorclock - -from leap.soledad.common import couch -from leap.soledad.common.document import ServerDocument -from leap.soledad.common.couch import errors - -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests.util import CouchDBTestCase -from leap.soledad.common.tests.util import make_local_db_and_target -from leap.soledad.common.tests.util import sync_via_synchronizer - -from leap.soledad.common.tests.u1db_tests import test_backends -from leap.soledad.common.tests.u1db_tests import DatabaseBaseTests - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_common_backend`. -# ----------------------------------------------------------------------------- - -class TestCouchBackendImpl(CouchDBTestCase): - - def test__allocate_doc_id(self): - db = couch.CouchDatabase.open_database( - urljoin( - 'http://localhost:' + str(self.couch_port), - ('test-%s' % uuid4().hex) - ), - create=True, - ensure_ddocs=True) - doc_id1 = db._allocate_doc_id() - self.assertTrue(doc_id1.startswith('D-')) - self.assertEqual(34, len(doc_id1)) - int(doc_id1[len('D-'):], 16) - self.assertNotEqual(doc_id1, db._allocate_doc_id()) - self.delete_db(db._dbname) - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_backends`. -# ----------------------------------------------------------------------------- - -def make_couch_database_for_test(test, replica_uid): - port = str(test.couch_port) - dbname = ('test-%s' % uuid4().hex) - db = couch.CouchDatabase.open_database( - urljoin('http://localhost:' + port, dbname), - create=True, - replica_uid=replica_uid or 'test', - ensure_ddocs=True) - test.addCleanup(test.delete_db, dbname) - return db - - -def copy_couch_database_for_test(test, db): - port = str(test.couch_port) - couch_url = 'http://localhost:' + port - new_dbname = db._dbname + '_copy' - new_db = couch.CouchDatabase.open_database( - urljoin(couch_url, new_dbname), - create=True, - replica_uid=db._replica_uid or 'test') - # copy all docs - session = couch.Session() - old_couch_db = Server(couch_url, session=session)[db._dbname] - new_couch_db = Server(couch_url, session=session)[new_dbname] - for doc_id in old_couch_db: - doc = old_couch_db.get(doc_id) - # bypass u1db_config document - if doc_id == 'u1db_config': - pass - # copy design docs - elif doc_id.startswith('_design'): - del doc['_rev'] - new_couch_db.save(doc) - # copy u1db docs - elif 'u1db_rev' in doc: - new_doc = { - '_id': doc['_id'], - 'u1db_transactions': doc['u1db_transactions'], - 'u1db_rev': doc['u1db_rev'] - } - attachments = [] - if ('u1db_conflicts' in doc): - new_doc['u1db_conflicts'] = doc['u1db_conflicts'] - for c_rev in doc['u1db_conflicts']: - attachments.append('u1db_conflict_%s' % c_rev) - new_couch_db.save(new_doc) - # save conflict data - attachments.append('u1db_content') - for att_name in attachments: - att = old_couch_db.get_attachment(doc_id, att_name) - if (att is not None): - new_couch_db.put_attachment(new_doc, att, - filename=att_name) - # cleanup connections to prevent file descriptor leaking - return new_db - - -def make_document_for_test(test, doc_id, rev, content, has_conflicts=False): - return ServerDocument( - doc_id, rev, content, has_conflicts=has_conflicts) - - -COUCH_SCENARIOS = [ - ('couch', {'make_database_for_test': make_couch_database_for_test, - 'copy_database_for_test': copy_couch_database_for_test, - 'make_document_for_test': make_document_for_test, }), -] - - -class CouchTests( - TestWithScenarios, test_backends.AllDatabaseTests, CouchDBTestCase): - - scenarios = COUCH_SCENARIOS - - -class SoledadBackendTests( - TestWithScenarios, - test_backends.LocalDatabaseTests, - CouchDBTestCase): - - scenarios = COUCH_SCENARIOS - - -class CouchValidateGenNTransIdTests( - TestWithScenarios, - test_backends.LocalDatabaseValidateGenNTransIdTests, - CouchDBTestCase): - - scenarios = COUCH_SCENARIOS - - -class CouchValidateSourceGenTests( - TestWithScenarios, - test_backends.LocalDatabaseValidateSourceGenTests, - CouchDBTestCase): - - scenarios = COUCH_SCENARIOS - - -class CouchWithConflictsTests( - TestWithScenarios, - test_backends.LocalDatabaseWithConflictsTests, - CouchDBTestCase): - - scenarios = COUCH_SCENARIOS - - -# Notice: the CouchDB backend does not have indexing capabilities, so we do -# not test indexing now. - -# class CouchIndexTests(test_backends.DatabaseIndexTests, CouchDBTestCase): -# -# scenarios = COUCH_SCENARIOS -# -# def tearDown(self): -# self.db.delete_database() -# test_backends.DatabaseIndexTests.tearDown(self) - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_sync`. -# ----------------------------------------------------------------------------- - -target_scenarios = [ - ('local', {'create_db_and_target': make_local_db_and_target}), ] - - -simple_doc = tests.simple_doc -nested_doc = tests.nested_doc - - -class SoledadBackendSyncTargetTests( - TestWithScenarios, - DatabaseBaseTests, - CouchDBTestCase): - - # TODO: implement _set_trace_hook(_shallow) in CouchSyncTarget so - # skipped tests can be succesfully executed. - - # whitebox true means self.db is the actual local db object - # against which the sync is performed - whitebox = True - - scenarios = (tests.multiply_scenarios(COUCH_SCENARIOS, target_scenarios)) - - def set_trace_hook(self, callback, shallow=False): - setter = (self.st._set_trace_hook if not shallow else - self.st._set_trace_hook_shallow) - try: - setter(callback) - except NotImplementedError: - self.skipTest("%s does not implement _set_trace_hook" - % (self.st.__class__.__name__,)) - - def setUp(self): - CouchDBTestCase.setUp(self) - # other stuff - self.db, self.st = self.create_db_and_target(self) - self.other_changes = [] - - def tearDown(self): - self.db.close() - CouchDBTestCase.tearDown(self) - - def receive_doc(self, doc, gen, trans_id): - self.other_changes.append( - (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) - - def test_sync_exchange_returns_many_new_docs(self): - # This test was replicated to allow dictionaries to be compared after - # JSON expansion (because one dictionary may have many different - # serialized representations). - doc = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(nested_doc) - self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) - new_gen, _ = self.st.sync_exchange( - [], 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) - self.assertEqual(2, new_gen) - self.assertEqual( - [(doc.doc_id, doc.rev, json.loads(simple_doc), 1), - (doc2.doc_id, doc2.rev, json.loads(nested_doc), 2)], - [c[:-3] + (json.loads(c[-3]), c[-2]) for c in self.other_changes]) - if self.whitebox: - self.assertEqual( - self.db._last_exchange_log['return'], - {'last_gen': 2, 'docs': - [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]}) - - def test_get_sync_target(self): - self.assertIsNot(None, self.st) - - def test_get_sync_info(self): - self.assertEqual( - ('test', 0, '', 0, ''), self.st.get_sync_info('other')) - - def test_create_doc_updates_sync_info(self): - self.assertEqual( - ('test', 0, '', 0, ''), self.st.get_sync_info('other')) - self.db.create_doc_from_json(simple_doc) - self.assertEqual(1, self.st.get_sync_info('other')[1]) - - def test_record_sync_info(self): - self.st.record_sync_info('replica', 10, 'T-transid') - self.assertEqual( - ('test', 0, '', 10, 'T-transid'), self.st.get_sync_info('replica')) - - def test_sync_exchange(self): - docs_by_gen = [ - (self.make_document('doc-id', 'replica:1', simple_doc), 10, - 'T-sid')] - new_gen, trans_id = self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertGetDoc(self.db, 'doc-id', 'replica:1', simple_doc, False) - self.assertTransactionLog(['doc-id'], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 1, last_trans_id), - (self.other_changes, new_gen, last_trans_id)) - self.assertEqual(10, self.st.get_sync_info('replica')[3]) - - def test_sync_exchange_deleted(self): - doc = self.db.create_doc_from_json('{}') - edit_rev = 'replica:1|' + doc.rev - docs_by_gen = [ - (self.make_document(doc.doc_id, edit_rev, None), 10, 'T-sid')] - new_gen, trans_id = self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertGetDocIncludeDeleted( - self.db, doc.doc_id, edit_rev, None, False) - self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 2, last_trans_id), - (self.other_changes, new_gen, trans_id)) - self.assertEqual(10, self.st.get_sync_info('replica')[3]) - - def test_sync_exchange_push_many(self): - docs_by_gen = [ - (self.make_document('doc-id', 'replica:1', simple_doc), 10, 'T-1'), - (self.make_document('doc-id2', 'replica:1', nested_doc), 11, - 'T-2')] - new_gen, trans_id = self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertGetDoc(self.db, 'doc-id', 'replica:1', simple_doc, False) - self.assertGetDoc(self.db, 'doc-id2', 'replica:1', nested_doc, False) - self.assertTransactionLog(['doc-id', 'doc-id2'], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 2, last_trans_id), - (self.other_changes, new_gen, trans_id)) - self.assertEqual(11, self.st.get_sync_info('replica')[3]) - - def test_sync_exchange_refuses_conflicts(self): - doc = self.db.create_doc_from_json(simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_doc = '{"key": "altval"}' - docs_by_gen = [ - (self.make_document(doc.doc_id, 'replica:1', new_doc), 10, - 'T-sid')] - new_gen, _ = self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id], self.db) - self.assertEqual( - (doc.doc_id, doc.rev, simple_doc, 1), self.other_changes[0][:-1]) - self.assertEqual(1, new_gen) - if self.whitebox: - self.assertEqual(self.db._last_exchange_log['return'], - {'last_gen': 1, 'docs': [(doc.doc_id, doc.rev)]}) - - def test_sync_exchange_ignores_convergence(self): - doc = self.db.create_doc_from_json(simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - gen, txid = self.db._get_generation_info() - docs_by_gen = [ - (self.make_document(doc.doc_id, doc.rev, simple_doc), 10, 'T-sid')] - new_gen, _ = self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=gen, - last_known_trans_id=txid, return_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id], self.db) - self.assertEqual(([], 1), (self.other_changes, new_gen)) - - def test_sync_exchange_returns_new_docs(self): - doc = self.db.create_doc_from_json(simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_gen, _ = self.st.sync_exchange( - [], 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id], self.db) - self.assertEqual( - (doc.doc_id, doc.rev, simple_doc, 1), self.other_changes[0][:-1]) - self.assertEqual(1, new_gen) - if self.whitebox: - self.assertEqual(self.db._last_exchange_log['return'], - {'last_gen': 1, 'docs': [(doc.doc_id, doc.rev)]}) - - def test_sync_exchange_returns_deleted_docs(self): - doc = self.db.create_doc_from_json(simple_doc) - self.db.delete_doc(doc) - self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) - new_gen, _ = self.st.sync_exchange( - [], 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) - self.assertEqual( - (doc.doc_id, doc.rev, None, 2), self.other_changes[0][:-1]) - self.assertEqual(2, new_gen) - if self.whitebox: - self.assertEqual(self.db._last_exchange_log['return'], - {'last_gen': 2, 'docs': [(doc.doc_id, doc.rev)]}) - - def test_sync_exchange_getting_newer_docs(self): - doc = self.db.create_doc_from_json(simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_doc = '{"key": "altval"}' - docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, - 'T-sid')] - new_gen, _ = self.st.sync_exchange( - docs_by_gen, 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) - self.assertEqual(([], 2), (self.other_changes, new_gen)) - - def test_sync_exchange_with_concurrent_updates_of_synced_doc(self): - expected = [] - - def before_whatschanged_cb(state): - if state != 'before whats_changed': - return - cont = '{"key": "cuncurrent"}' - conc_rev = self.db.put_doc( - self.make_document(doc.doc_id, 'test:1|z:2', cont)) - expected.append((doc.doc_id, conc_rev, cont, 3)) - - self.set_trace_hook(before_whatschanged_cb) - doc = self.db.create_doc_from_json(simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_doc = '{"key": "altval"}' - docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, - 'T-sid')] - new_gen, _ = self.st.sync_exchange( - docs_by_gen, 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertEqual(expected, [c[:-1] for c in self.other_changes]) - self.assertEqual(3, new_gen) - - def test_sync_exchange_with_concurrent_updates(self): - - def after_whatschanged_cb(state): - if state != 'after whats_changed': - return - self.db.create_doc_from_json('{"new": "doc"}') - - self.set_trace_hook(after_whatschanged_cb) - doc = self.db.create_doc_from_json(simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_doc = '{"key": "altval"}' - docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, - 'T-sid')] - new_gen, _ = self.st.sync_exchange( - docs_by_gen, 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertEqual(([], 2), (self.other_changes, new_gen)) - - def test_sync_exchange_converged_handling(self): - doc = self.db.create_doc_from_json(simple_doc) - docs_by_gen = [ - (self.make_document('new', 'other:1', '{}'), 4, 'T-foo'), - (self.make_document(doc.doc_id, doc.rev, doc.get_json()), 5, - 'T-bar')] - new_gen, _ = self.st.sync_exchange( - docs_by_gen, 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertEqual(([], 2), (self.other_changes, new_gen)) - - def test_sync_exchange_detect_incomplete_exchange(self): - def before_get_docs_explode(state): - if state != 'before get_docs': - return - raise u1db_errors.U1DBError("fail") - self.set_trace_hook(before_get_docs_explode) - # suppress traceback printing in the wsgiref server - # self.patch(simple_server.ServerHandler, - # 'log_exception', lambda h, exc_info: None) - doc = self.db.create_doc_from_json(simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - self.assertRaises( - (u1db_errors.U1DBError, u1db_errors.BrokenSyncStream), - self.st.sync_exchange, [], 'other-replica', - last_known_generation=0, last_known_trans_id=None, - return_doc_cb=self.receive_doc) - - def test_sync_exchange_doc_ids(self): - sync_exchange_doc_ids = getattr(self.st, 'sync_exchange_doc_ids', None) - if sync_exchange_doc_ids is None: - self.skipTest("sync_exchange_doc_ids not implemented") - db2 = self.create_database('test2') - doc = db2.create_doc_from_json(simple_doc) - new_gen, trans_id = sync_exchange_doc_ids( - db2, [(doc.doc_id, 10, 'T-sid')], 0, None, - return_doc_cb=self.receive_doc) - self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) - self.assertTransactionLog([doc.doc_id], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 1, last_trans_id), - (self.other_changes, new_gen, trans_id)) - self.assertEqual(10, self.st.get_sync_info(db2._replica_uid)[3]) - - def test__set_trace_hook(self): - called = [] - - def cb(state): - called.append(state) - - self.set_trace_hook(cb) - self.st.sync_exchange([], 'replica', 0, None, self.receive_doc) - self.st.record_sync_info('replica', 0, 'T-sid') - self.assertEqual(['before whats_changed', - 'after whats_changed', - 'before get_docs', - 'record_sync_info', - ], - called) - - def test__set_trace_hook_shallow(self): - st_trace_shallow = self.st._set_trace_hook_shallow - target_st_trace_shallow = SyncTarget._set_trace_hook_shallow - same_meth = st_trace_shallow == self.st._set_trace_hook - same_fun = st_trace_shallow.im_func == target_st_trace_shallow.im_func - if (same_meth or same_fun): - # shallow same as full - expected = ['before whats_changed', - 'after whats_changed', - 'before get_docs', - 'record_sync_info', - ] - else: - expected = ['sync_exchange', 'record_sync_info'] - - called = [] - - def cb(state): - called.append(state) - - self.set_trace_hook(cb, shallow=True) - self.st.sync_exchange([], 'replica', 0, None, self.receive_doc) - self.st.record_sync_info('replica', 0, 'T-sid') - self.assertEqual(expected, called) - -sync_scenarios = [] -for name, scenario in COUCH_SCENARIOS: - scenario = dict(scenario) - scenario['do_sync'] = sync_via_synchronizer - sync_scenarios.append((name, scenario)) - scenario = dict(scenario) - - -class SoledadBackendSyncTests( - TestWithScenarios, - DatabaseBaseTests, - CouchDBTestCase): - - scenarios = sync_scenarios - - def create_database(self, replica_uid, sync_role=None): - if replica_uid == 'test' and sync_role is None: - # created up the chain by base class but unused - return None - db = self.create_database_for_role(replica_uid, sync_role) - if sync_role: - self._use_tracking[db] = (replica_uid, sync_role) - return db - - def create_database_for_role(self, replica_uid, sync_role): - # hook point for reuse - return DatabaseBaseTests.create_database(self, replica_uid) - - def copy_database(self, db, sync_role=None): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES - # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST - # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS - # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND - # NINJA TO YOUR HOUSE. - db_copy = self.copy_database_for_test(self, db) - name, orig_sync_role = self._use_tracking[db] - self._use_tracking[db_copy] = ( - name + '(copy)', sync_role or orig_sync_role) - return db_copy - - def sync(self, db_from, db_to, trace_hook=None, - trace_hook_shallow=None): - from_name, from_sync_role = self._use_tracking[db_from] - to_name, to_sync_role = self._use_tracking[db_to] - if from_sync_role not in ('source', 'both'): - raise Exception("%s marked for %s use but used as source" % - (from_name, from_sync_role)) - if to_sync_role not in ('target', 'both'): - raise Exception("%s marked for %s use but used as target" % - (to_name, to_sync_role)) - return self.do_sync(self, db_from, db_to, trace_hook, - trace_hook_shallow) - - def setUp(self): - self.db = None - self.db1 = None - self.db2 = None - self.db3 = None - self.db1_copy = None - self.db2_copy = None - self._use_tracking = {} - DatabaseBaseTests.setUp(self) - - def tearDown(self): - for db in [ - self.db, self.db1, self.db2, - self.db3, self.db1_copy, self.db2_copy - ]: - if db is not None: - self.delete_db(db._dbname) - db.close() - DatabaseBaseTests.tearDown(self) - - def assertLastExchangeLog(self, db, expected): - log = getattr(db, '_last_exchange_log', None) - if log is None: - return - self.assertEqual(expected, log) - - def test_sync_tracks_db_generation_of_other(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.assertEqual(0, self.sync(self.db1, self.db2)) - self.assertEqual( - (0, ''), self.db1._get_replica_gen_and_trans_id('test2')) - self.assertEqual( - (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [], 'last_known_gen': 0}, - 'return': {'docs': [], 'last_gen': 0}}) - - def test_sync_autoresolves(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc1 = self.db1.create_doc_from_json(simple_doc, doc_id='doc') - rev1 = doc1.rev - doc2 = self.db2.create_doc_from_json(simple_doc, doc_id='doc') - rev2 = doc2.rev - self.sync(self.db1, self.db2) - doc = self.db1.get_doc('doc') - self.assertFalse(doc.has_conflicts) - self.assertEqual(doc.rev, self.db2.get_doc('doc').rev) - v = vectorclock.VectorClockRev(doc.rev) - self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev1))) - self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev2))) - - def test_sync_autoresolves_moar(self): - # here we test that when a database that has a conflicted document is - # the source of a sync, and the target database has a revision of the - # conflicted document that is newer than the source database's, and - # that target's database's document's content is the same as the - # source's document's conflict's, the source's document's conflict gets - # autoresolved, and the source's document's revision bumped. - # - # idea is as follows: - # A B - # a1 - - # `-------> - # a1 a1 - # v v - # a2 a1b1 - # `-------> - # a1b1+a2 a1b1 - # v - # a1b1+a2 a1b2 (a1b2 has same content as a2) - # `-------> - # a3b2 a1b2 (autoresolved) - # `-------> - # a3b2 a3b2 - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(simple_doc, doc_id='doc') - self.sync(self.db1, self.db2) - for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: - doc = db.get_doc('doc') - doc.set_json(content) - db.put_doc(doc) - self.sync(self.db1, self.db2) - # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict - doc = self.db1.get_doc('doc') - rev1 = doc.rev - self.assertTrue(doc.has_conflicts) - # set db2 to have a doc of {} (same as db1 before the conflict) - doc = self.db2.get_doc('doc') - doc.set_json('{}') - self.db2.put_doc(doc) - rev2 = doc.rev - # sync it across - self.sync(self.db1, self.db2) - # tadaa! - doc = self.db1.get_doc('doc') - self.assertFalse(doc.has_conflicts) - vec1 = vectorclock.VectorClockRev(rev1) - vec2 = vectorclock.VectorClockRev(rev2) - vec3 = vectorclock.VectorClockRev(doc.rev) - self.assertTrue(vec3.is_newer(vec1)) - self.assertTrue(vec3.is_newer(vec2)) - # because the conflict is on the source, sync it another time - self.sync(self.db1, self.db2) - # make sure db2 now has the exact same thing - self.assertEqual(self.db1.get_doc('doc'), self.db2.get_doc('doc')) - - def test_sync_autoresolves_moar_backwards(self): - # here we test that when a database that has a conflicted document is - # the target of a sync, and the source database has a revision of the - # conflicted document that is newer than the target database's, and - # that source's database's document's content is the same as the - # target's document's conflict's, the target's document's conflict gets - # autoresolved, and the document's revision bumped. - # - # idea is as follows: - # A B - # a1 - - # `-------> - # a1 a1 - # v v - # a2 a1b1 - # `-------> - # a1b1+a2 a1b1 - # v - # a1b1+a2 a1b2 (a1b2 has same content as a2) - # <-------' - # a3b2 a3b2 (autoresolved and propagated) - self.db1 = self.create_database('test1', 'both') - self.db2 = self.create_database('test2', 'both') - self.db1.create_doc_from_json(simple_doc, doc_id='doc') - self.sync(self.db1, self.db2) - for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: - doc = db.get_doc('doc') - doc.set_json(content) - db.put_doc(doc) - self.sync(self.db1, self.db2) - # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict - doc = self.db1.get_doc('doc') - rev1 = doc.rev - self.assertTrue(doc.has_conflicts) - revc = self.db1.get_doc_conflicts('doc')[-1].rev - # set db2 to have a doc of {} (same as db1 before the conflict) - doc = self.db2.get_doc('doc') - doc.set_json('{}') - self.db2.put_doc(doc) - rev2 = doc.rev - # sync it across - self.sync(self.db2, self.db1) - # tadaa! - doc = self.db1.get_doc('doc') - self.assertFalse(doc.has_conflicts) - vec1 = vectorclock.VectorClockRev(rev1) - vec2 = vectorclock.VectorClockRev(rev2) - vec3 = vectorclock.VectorClockRev(doc.rev) - vecc = vectorclock.VectorClockRev(revc) - self.assertTrue(vec3.is_newer(vec1)) - self.assertTrue(vec3.is_newer(vec2)) - self.assertTrue(vec3.is_newer(vecc)) - # make sure db2 now has the exact same thing - self.assertEqual(self.db1.get_doc('doc'), self.db2.get_doc('doc')) - - def test_sync_autoresolves_moar_backwards_three(self): - # same as autoresolves_moar_backwards, but with three databases (note - # all the syncs go in the same direction -- this is a more natural - # scenario): - # - # A B C - # a1 - - - # `-------> - # a1 a1 - - # `-------> - # a1 a1 a1 - # v v - # a2 a1b1 a1 - # `-------------------> - # a2 a1b1 a2 - # `-------> - # a2+a1b1 a2 - # v - # a2 a2+a1b1 a2c1 (same as a1b1) - # `-------------------> - # a2c1 a2+a1b1 a2c1 - # `-------> - # a2b2c1 a2b2c1 a2c1 - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'both') - self.db3 = self.create_database('test3', 'target') - self.db1.create_doc_from_json(simple_doc, doc_id='doc') - self.sync(self.db1, self.db2) - self.sync(self.db2, self.db3) - for db, content in [(self.db2, '{"hi": 42}'), - (self.db1, '{}'), - ]: - doc = db.get_doc('doc') - doc.set_json(content) - db.put_doc(doc) - self.sync(self.db1, self.db3) - self.sync(self.db2, self.db3) - # db2 and db3 now both have a doc of {}, but db2 has a - # conflict - doc = self.db2.get_doc('doc') - self.assertTrue(doc.has_conflicts) - revc = self.db2.get_doc_conflicts('doc')[-1].rev - self.assertEqual('{}', doc.get_json()) - self.assertEqual(self.db3.get_doc('doc').get_json(), doc.get_json()) - self.assertEqual(self.db3.get_doc('doc').rev, doc.rev) - # set db3 to have a doc of {hi:42} (same as db2 before the conflict) - doc = self.db3.get_doc('doc') - doc.set_json('{"hi": 42}') - self.db3.put_doc(doc) - rev3 = doc.rev - # sync it across to db1 - self.sync(self.db1, self.db3) - # db1 now has hi:42, with a rev that is newer than db2's doc - doc = self.db1.get_doc('doc') - rev1 = doc.rev - self.assertFalse(doc.has_conflicts) - self.assertEqual('{"hi": 42}', doc.get_json()) - VCR = vectorclock.VectorClockRev - self.assertTrue(VCR(rev1).is_newer(VCR(self.db2.get_doc('doc').rev))) - # so sync it to db2 - self.sync(self.db1, self.db2) - # tadaa! - doc = self.db2.get_doc('doc') - self.assertFalse(doc.has_conflicts) - # db2's revision of the document is strictly newer than db1's before - # the sync, and db3's before that sync way back when - self.assertTrue(VCR(doc.rev).is_newer(VCR(rev1))) - self.assertTrue(VCR(doc.rev).is_newer(VCR(rev3))) - self.assertTrue(VCR(doc.rev).is_newer(VCR(revc))) - # make sure both dbs now have the exact same thing - self.assertEqual(self.db1.get_doc('doc'), self.db2.get_doc('doc')) - - def test_sync_puts_changes(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db1.create_doc_from_json(simple_doc) - self.assertEqual(1, self.sync(self.db1, self.db2)) - self.assertGetDoc(self.db2, doc.doc_id, doc.rev, simple_doc, False) - self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) - self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [(doc.doc_id, doc.rev)], - 'source_uid': 'test1', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': {'docs': [], 'last_gen': 1}}) - - def test_sync_pulls_changes(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db2.create_doc_from_json(simple_doc) - self.assertEqual(0, self.sync(self.db1, self.db2)) - self.assertGetDoc(self.db1, doc.doc_id, doc.rev, simple_doc, False) - self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) - self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [], 'last_known_gen': 0}, - 'return': {'docs': [(doc.doc_id, doc.rev)], - 'last_gen': 1}}) - self.assertGetDoc(self.db2, doc.doc_id, doc.rev, simple_doc, False) - - def test_sync_pulling_doesnt_update_other_if_changed(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db2.create_doc_from_json(simple_doc) - # After the local side has sent its list of docs, before we start - # receiving the "targets" response, we update the local database with a - # new record. - # When we finish synchronizing, we can notice that something locally - # was updated, and we cannot tell c2 our new updated generation - - def before_get_docs(state): - if state != 'before get_docs': - return - self.db1.create_doc_from_json(simple_doc) - - self.assertEqual(0, self.sync(self.db1, self.db2, - trace_hook=before_get_docs)) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [], 'last_known_gen': 0}, - 'return': {'docs': [(doc.doc_id, doc.rev)], - 'last_gen': 1}}) - self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) - # c2 should not have gotten a '_record_sync_info' call, because the - # local database had been updated more than just by the messages - # returned from c2. - self.assertEqual( - (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) - - def test_sync_doesnt_update_other_if_nothing_pulled(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(simple_doc) - - def no_record_sync_info(state): - if state != 'record_sync_info': - return - self.fail('SyncTarget.record_sync_info was called') - self.assertEqual(1, self.sync(self.db1, self.db2, - trace_hook_shallow=no_record_sync_info)) - self.assertEqual( - 1, - self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)[0]) - - def test_sync_ignores_convergence(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'both') - doc = self.db1.create_doc_from_json(simple_doc) - self.db3 = self.create_database('test3', 'target') - self.assertEqual(1, self.sync(self.db1, self.db3)) - self.assertEqual(0, self.sync(self.db2, self.db3)) - self.assertEqual(1, self.sync(self.db1, self.db2)) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [(doc.doc_id, doc.rev)], - 'source_uid': 'test1', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': {'docs': [], 'last_gen': 1}}) - - def test_sync_ignores_superseded(self): - self.db1 = self.create_database('test1', 'both') - self.db2 = self.create_database('test2', 'both') - doc = self.db1.create_doc_from_json(simple_doc) - doc_rev1 = doc.rev - self.db3 = self.create_database('test3', 'target') - self.sync(self.db1, self.db3) - self.sync(self.db2, self.db3) - new_content = '{"key": "altval"}' - doc.set_json(new_content) - self.db1.put_doc(doc) - doc_rev2 = doc.rev - self.sync(self.db2, self.db1) - self.assertLastExchangeLog( - self.db1, - {'receive': {'docs': [(doc.doc_id, doc_rev1)], - 'source_uid': 'test2', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': {'docs': [(doc.doc_id, doc_rev2)], - 'last_gen': 2}}) - self.assertGetDoc(self.db1, doc.doc_id, doc_rev2, new_content, False) - - def test_sync_sees_remote_conflicted(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc1 = self.db1.create_doc_from_json(simple_doc) - doc_id = doc1.doc_id - doc1_rev = doc1.rev - new_doc = '{"key": "altval"}' - doc2 = self.db2.create_doc_from_json(new_doc, doc_id=doc_id) - doc2_rev = doc2.rev - self.assertTransactionLog([doc1.doc_id], self.db1) - self.sync(self.db1, self.db2) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [(doc_id, doc1_rev)], - 'source_uid': 'test1', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': {'docs': [(doc_id, doc2_rev)], - 'last_gen': 1}}) - self.assertTransactionLog([doc_id, doc_id], self.db1) - self.assertGetDoc(self.db1, doc_id, doc2_rev, new_doc, True) - self.assertGetDoc(self.db2, doc_id, doc2_rev, new_doc, False) - - def test_sync_sees_remote_delete_conflicted(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc1 = self.db1.create_doc_from_json(simple_doc) - doc_id = doc1.doc_id - self.sync(self.db1, self.db2) - doc2 = self.make_document(doc1.doc_id, doc1.rev, doc1.get_json()) - new_doc = '{"key": "altval"}' - doc1.set_json(new_doc) - self.db1.put_doc(doc1) - self.db2.delete_doc(doc2) - self.assertTransactionLog([doc_id, doc_id], self.db1) - self.sync(self.db1, self.db2) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [(doc_id, doc1.rev)], - 'source_uid': 'test1', - 'source_gen': 2, 'last_known_gen': 1}, - 'return': {'docs': [(doc_id, doc2.rev)], - 'last_gen': 2}}) - self.assertTransactionLog([doc_id, doc_id, doc_id], self.db1) - self.assertGetDocIncludeDeleted(self.db1, doc_id, doc2.rev, None, True) - self.assertGetDocIncludeDeleted( - self.db2, doc_id, doc2.rev, None, False) - - def test_sync_local_race_conflicted(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db1.create_doc_from_json(simple_doc) - doc_id = doc.doc_id - doc1_rev = doc.rev - self.sync(self.db1, self.db2) - content1 = '{"key": "localval"}' - content2 = '{"key": "altval"}' - doc.set_json(content2) - self.db2.put_doc(doc) - doc2_rev2 = doc.rev - triggered = [] - - def after_whatschanged(state): - if state != 'after whats_changed': - return - triggered.append(True) - doc = self.make_document(doc_id, doc1_rev, content1) - self.db1.put_doc(doc) - - self.sync(self.db1, self.db2, trace_hook=after_whatschanged) - self.assertEqual([True], triggered) - self.assertGetDoc(self.db1, doc_id, doc2_rev2, content2, True) - - def test_sync_propagates_deletes(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'both') - doc1 = self.db1.create_doc_from_json(simple_doc) - doc_id = doc1.doc_id - self.sync(self.db1, self.db2) - self.db3 = self.create_database('test3', 'target') - self.sync(self.db1, self.db3) - self.db1.delete_doc(doc1) - deleted_rev = doc1.rev - self.sync(self.db1, self.db2) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [(doc_id, deleted_rev)], - 'source_uid': 'test1', - 'source_gen': 2, 'last_known_gen': 1}, - 'return': {'docs': [], 'last_gen': 2}}) - self.assertGetDocIncludeDeleted( - self.db1, doc_id, deleted_rev, None, False) - self.assertGetDocIncludeDeleted( - self.db2, doc_id, deleted_rev, None, False) - self.sync(self.db2, self.db3) - self.assertLastExchangeLog( - self.db3, - {'receive': {'docs': [(doc_id, deleted_rev)], - 'source_uid': 'test2', - 'source_gen': 2, 'last_known_gen': 0}, - 'return': {'docs': [], 'last_gen': 2}}) - self.assertGetDocIncludeDeleted( - self.db3, doc_id, deleted_rev, None, False) - - def test_sync_propagates_deletes_2(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json('{"a": "1"}', doc_id='the-doc') - self.sync(self.db1, self.db2) - doc1_2 = self.db2.get_doc('the-doc') - self.db2.delete_doc(doc1_2) - self.sync(self.db1, self.db2) - self.assertGetDocIncludeDeleted( - self.db1, 'the-doc', doc1_2.rev, None, False) - - def test_sync_propagates_resolution(self): - self.db1 = self.create_database('test1', 'both') - self.db2 = self.create_database('test2', 'both') - doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') - self.db3 = self.create_database('test3', 'both') - self.sync(self.db2, self.db1) - self.assertEqual( - self.db1._get_generation_info(), - self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)) - self.assertEqual( - self.db2._get_generation_info(), - self.db1._get_replica_gen_and_trans_id(self.db2._replica_uid)) - self.sync(self.db3, self.db1) - # update on 2 - doc2 = self.make_document('the-doc', doc1.rev, '{"a": 2}') - self.db2.put_doc(doc2) - self.sync(self.db2, self.db3) - self.assertEqual(self.db3.get_doc('the-doc').rev, doc2.rev) - # update on 1 - doc1.set_json('{"a": 3}') - self.db1.put_doc(doc1) - # conflicts - self.sync(self.db2, self.db1) - self.sync(self.db3, self.db1) - self.assertTrue(self.db2.get_doc('the-doc').has_conflicts) - self.assertTrue(self.db3.get_doc('the-doc').has_conflicts) - # resolve - conflicts = self.db2.get_doc_conflicts('the-doc') - doc4 = self.make_document('the-doc', None, '{"a": 4}') - revs = [doc.rev for doc in conflicts] - self.db2.resolve_doc(doc4, revs) - doc2 = self.db2.get_doc('the-doc') - self.assertEqual(doc4.get_json(), doc2.get_json()) - self.assertFalse(doc2.has_conflicts) - self.sync(self.db2, self.db3) - doc3 = self.db3.get_doc('the-doc') - self.assertEqual(doc4.get_json(), doc3.get_json()) - self.assertFalse(doc3.has_conflicts) - - def test_sync_supersedes_conflicts(self): - self.db1 = self.create_database('test1', 'both') - self.db2 = self.create_database('test2', 'target') - self.db3 = self.create_database('test3', 'both') - doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') - self.db2.create_doc_from_json('{"b": 1}', doc_id='the-doc') - self.db3.create_doc_from_json('{"c": 1}', doc_id='the-doc') - self.sync(self.db3, self.db1) - self.assertEqual( - self.db1._get_generation_info(), - self.db3._get_replica_gen_and_trans_id(self.db1._replica_uid)) - self.assertEqual( - self.db3._get_generation_info(), - self.db1._get_replica_gen_and_trans_id(self.db3._replica_uid)) - self.sync(self.db3, self.db2) - self.assertEqual( - self.db2._get_generation_info(), - self.db3._get_replica_gen_and_trans_id(self.db2._replica_uid)) - self.assertEqual( - self.db3._get_generation_info(), - self.db2._get_replica_gen_and_trans_id(self.db3._replica_uid)) - self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) - doc1.set_json('{"a": 2}') - self.db1.put_doc(doc1) - self.sync(self.db3, self.db1) - # original doc1 should have been removed from conflicts - self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) - - def test_sync_stops_after_get_sync_info(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc) - self.sync(self.db1, self.db2) - - def put_hook(state): - self.fail("Tracehook triggered for %s" % (state,)) - - self.sync(self.db1, self.db2, trace_hook_shallow=put_hook) - - def test_sync_detects_identical_replica_uid(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test1', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') - self.assertRaises( - u1db_errors.InvalidReplicaUID, self.sync, self.db1, self.db2) - - def test_sync_detects_rollback_in_source(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') - self.sync(self.db1, self.db2) - self.db1_copy = self.copy_database(self.db1) - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.sync(self.db1, self.db2) - self.assertRaises( - u1db_errors.InvalidGeneration, self.sync, self.db1_copy, self.db2) - - def test_sync_detects_rollback_in_target(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.db2_copy = self.copy_database(self.db2) - self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.sync(self.db1, self.db2) - self.assertRaises( - u1db_errors.InvalidGeneration, self.sync, self.db1, self.db2_copy) - - def test_sync_detects_diverged_source(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db3 = self.copy_database(self.db1) - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.db3.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.assertRaises( - u1db_errors.InvalidTransactionId, self.sync, self.db3, self.db2) - - def test_sync_detects_diverged_target(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db3 = self.copy_database(self.db2) - self.db3.create_doc_from_json(tests.nested_doc, doc_id="divergent") - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.assertRaises( - u1db_errors.InvalidTransactionId, self.sync, self.db1, self.db3) - - def test_sync_detects_rollback_and_divergence_in_source(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') - self.sync(self.db1, self.db2) - self.db1_copy = self.copy_database(self.db1) - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.sync(self.db1, self.db2) - self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.assertRaises( - u1db_errors.InvalidTransactionId, self.sync, - self.db1_copy, self.db2) - - def test_sync_detects_rollback_and_divergence_in_target(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.db2_copy = self.copy_database(self.db2) - self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.sync(self.db1, self.db2) - self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.assertRaises( - u1db_errors.InvalidTransactionId, self.sync, - self.db1, self.db2_copy) - - def test_optional_sync_preserve_json(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - cont1 = '{"a": 2}' - cont2 = '{"b": 3}' - self.db1.create_doc_from_json(cont1, doc_id="1") - self.db2.create_doc_from_json(cont2, doc_id="2") - self.sync(self.db1, self.db2) - self.assertEqual(cont1, self.db2.get_doc("1").get_json()) - self.assertEqual(cont2, self.db1.get_doc("2").get_json()) - - -class SoledadBackendExceptionsTests(CouchDBTestCase): - - def setUp(self): - CouchDBTestCase.setUp(self) - - def create_db(self, ensure=True, dbname=None): - if not dbname: - dbname = ('test-%s' % uuid4().hex) - if dbname not in self.couch_server: - self.couch_server.create(dbname) - self.db = couch.CouchDatabase( - ('http://127.0.0.1:%d' % self.couch_port), - dbname, - ensure_ddocs=ensure) - - def tearDown(self): - self.db.delete_database() - self.db.close() - CouchDBTestCase.tearDown(self) - - def test_missing_design_doc_raises(self): - """ - Test that all methods that access design documents will raise if the - design docs are not present. - """ - self.create_db(ensure=False) - # get_generation_info() - self.assertRaises( - errors.MissingDesignDocError, - self.db.get_generation_info) - # get_trans_id_for_gen() - self.assertRaises( - errors.MissingDesignDocError, - self.db.get_trans_id_for_gen, 1) - # get_transaction_log() - self.assertRaises( - errors.MissingDesignDocError, - self.db.get_transaction_log) - # whats_changed() - self.assertRaises( - errors.MissingDesignDocError, - self.db.whats_changed) - - def test_missing_design_doc_functions_raises(self): - """ - Test that all methods that access design documents list functions - will raise if the functions are not present. - """ - self.create_db(ensure=True) - # erase views from _design/transactions - transactions = self.db._database['_design/transactions'] - transactions['lists'] = {} - self.db._database.save(transactions) - # get_generation_info() - self.assertRaises( - errors.MissingDesignDocListFunctionError, - self.db.get_generation_info) - # get_trans_id_for_gen() - self.assertRaises( - errors.MissingDesignDocListFunctionError, - self.db.get_trans_id_for_gen, 1) - # whats_changed() - self.assertRaises( - errors.MissingDesignDocListFunctionError, - self.db.whats_changed) - - def test_absent_design_doc_functions_raises(self): - """ - Test that all methods that access design documents list functions - will raise if the functions are not present. - """ - self.create_db(ensure=True) - # erase views from _design/transactions - transactions = self.db._database['_design/transactions'] - del transactions['lists'] - self.db._database.save(transactions) - # get_generation_info() - self.assertRaises( - errors.MissingDesignDocListFunctionError, - self.db.get_generation_info) - # _get_trans_id_for_gen() - self.assertRaises( - errors.MissingDesignDocListFunctionError, - self.db.get_trans_id_for_gen, 1) - # whats_changed() - self.assertRaises( - errors.MissingDesignDocListFunctionError, - self.db.whats_changed) - - def test_missing_design_doc_named_views_raises(self): - """ - Test that all methods that access design documents' named views will - raise if the views are not present. - """ - self.create_db(ensure=True) - # erase views from _design/docs - docs = self.db._database['_design/docs'] - del docs['views'] - self.db._database.save(docs) - # erase views from _design/syncs - syncs = self.db._database['_design/syncs'] - del syncs['views'] - self.db._database.save(syncs) - # erase views from _design/transactions - transactions = self.db._database['_design/transactions'] - del transactions['views'] - self.db._database.save(transactions) - # get_generation_info() - self.assertRaises( - errors.MissingDesignDocNamedViewError, - self.db.get_generation_info) - # _get_trans_id_for_gen() - self.assertRaises( - errors.MissingDesignDocNamedViewError, - self.db.get_trans_id_for_gen, 1) - # _get_transaction_log() - self.assertRaises( - errors.MissingDesignDocNamedViewError, - self.db.get_transaction_log) - # whats_changed() - self.assertRaises( - errors.MissingDesignDocNamedViewError, - self.db.whats_changed) - - def test_deleted_design_doc_raises(self): - """ - Test that all methods that access design documents will raise if the - design docs are not present. - """ - self.create_db(ensure=True) - # delete _design/docs - del self.db._database['_design/docs'] - # delete _design/syncs - del self.db._database['_design/syncs'] - # delete _design/transactions - del self.db._database['_design/transactions'] - # get_generation_info() - self.assertRaises( - errors.MissingDesignDocDeletedError, - self.db.get_generation_info) - # get_trans_id_for_gen() - self.assertRaises( - errors.MissingDesignDocDeletedError, - self.db.get_trans_id_for_gen, 1) - # get_transaction_log() - self.assertRaises( - errors.MissingDesignDocDeletedError, - self.db.get_transaction_log) - # whats_changed() - self.assertRaises( - errors.MissingDesignDocDeletedError, - self.db.whats_changed) - - def test_ensure_ddoc_independently(self): - """ - Test that a missing ddocs other than _design/docs will be ensured - even if _design/docs is there. - """ - self.create_db(ensure=True) - del self.db._database['_design/transactions'] - self.assertRaises( - errors.MissingDesignDocDeletedError, - self.db.get_transaction_log) - self.create_db(ensure=True, dbname=self.db._dbname) - self.db.get_transaction_log() - - def test_ensure_security_doc(self): - """ - Ensure_security creates a _security ddoc to ensure that only soledad - will have the lowest privileged access to an user db. - """ - self.create_db(ensure=False) - self.assertFalse(self.db._database.resource.get_json('_security')[2]) - self.db.ensure_security_ddoc() - security_ddoc = self.db._database.resource.get_json('_security')[2] - self.assertIn('admins', security_ddoc) - self.assertFalse(security_ddoc['admins']['names']) - self.assertIn('members', security_ddoc) - self.assertIn('soledad', security_ddoc['members']['names']) - - def test_ensure_security_from_configuration(self): - """ - Given a configuration, follow it to create the security document - """ - self.create_db(ensure=False) - configuration = {'members': ['user1', 'user2'], - 'members_roles': ['role1', 'role2'], - 'admins': ['admin'], - 'admins_roles': ['administrators'] - } - self.db.ensure_security_ddoc(configuration) - - security_ddoc = self.db._database.resource.get_json('_security')[2] - self.assertEquals(configuration['admins'], - security_ddoc['admins']['names']) - self.assertEquals(configuration['admins_roles'], - security_ddoc['admins']['roles']) - self.assertEquals(configuration['members'], - security_ddoc['members']['names']) - self.assertEquals(configuration['members_roles'], - security_ddoc['members']['roles']) - - -class DatabaseNameValidationTest(unittest.TestCase): - - def test_database_name_validation(self): - inject = couch.state.is_db_name_valid("user-deadbeef | cat /secret") - self.assertFalse(inject) - self.assertTrue(couch.state.is_db_name_valid("user-cafe1337")) - - -class CommandBasedDBCreationTest(unittest.TestCase): - - def test_ensure_db_using_custom_command(self): - state = couch.state.CouchServerState("url", create_cmd="echo") - mock_db = Mock() - mock_db.replica_uid = 'replica_uid' - state.open_database = Mock(return_value=mock_db) - db, replica_uid = state.ensure_database("user-1337") # works - self.assertEquals(mock_db, db) - self.assertEquals(mock_db.replica_uid, replica_uid) - - def test_raises_unauthorized_on_failure(self): - state = couch.state.CouchServerState("url", create_cmd="inexistent") - self.assertRaises(u1db_errors.Unauthorized, - state.ensure_database, "user-1337") - - def test_raises_unauthorized_by_default(self): - state = couch.state.CouchServerState("url") - self.assertRaises(u1db_errors.Unauthorized, - state.ensure_database, "user-1337") diff --git a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py b/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py deleted file mode 100644 index 8cd3ae08..00000000 --- a/common/src/leap/soledad/common/tests/test_couch_operations_atomicity.py +++ /dev/null @@ -1,371 +0,0 @@ -# -*- coding: utf-8 -*- -# test_couch_operations_atomicity.py -# Copyright (C) 2013, 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Test atomicity of couch operations. -""" -import os -import tempfile -import threading - -from urlparse import urljoin -from twisted.internet import defer -from uuid import uuid4 - -from leap.soledad.client import Soledad -from leap.soledad.common.couch.state import CouchServerState -from leap.soledad.common.couch import CouchDatabase - -from leap.soledad.common.tests.util import ( - make_token_soledad_app, - make_soledad_document_for_test, - soledad_sync_target, -) -from leap.soledad.common.tests.test_couch import CouchDBTestCase -from leap.soledad.common.tests.u1db_tests import TestCaseWithServer - - -REPEAT_TIMES = 20 - - -class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): - - @staticmethod - def make_app_after_state(state): - return make_token_soledad_app(state) - - make_document_for_test = make_soledad_document_for_test - - sync_target = soledad_sync_target - - def _soledad_instance(self, user=None, passphrase=u'123', - prefix='', - secrets_path='secrets.json', - local_db_path='soledad.u1db', server_url='', - cert_file=None, auth_token=None): - """ - Instantiate Soledad. - """ - user = user or self.user - - # this callback ensures we save a document which is sent to the shared - # db. - def _put_doc_side_effect(doc): - self._doc_put = doc - - soledad = Soledad( - user, - passphrase, - secrets_path=os.path.join(self.tempdir, prefix, secrets_path), - local_db_path=os.path.join( - self.tempdir, prefix, local_db_path), - server_url=server_url, - cert_file=cert_file, - auth_token=auth_token, - shared_db=self.get_default_shared_mock(_put_doc_side_effect)) - self.addCleanup(soledad.close) - return soledad - - def make_app(self): - self.request_state = CouchServerState(self.couch_url) - return self.make_app_after_state(self.request_state) - - def setUp(self): - TestCaseWithServer.setUp(self) - CouchDBTestCase.setUp(self) - self.user = ('user-%s' % uuid4().hex) - self.db = CouchDatabase.open_database( - urljoin(self.couch_url, 'user-' + self.user), - create=True, - replica_uid='replica', - ensure_ddocs=True) - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.startTwistedServer() - - def tearDown(self): - self.db.delete_database() - self.db.close() - CouchDBTestCase.tearDown(self) - TestCaseWithServer.tearDown(self) - - # - # Sequential tests - # - - def test_correct_transaction_log_after_sequential_puts(self): - """ - Assert that the transaction_log increases accordingly with sequential - puts. - """ - doc = self.db.create_doc({'ops': 0}) - docs = [doc.doc_id] - for i in range(0, REPEAT_TIMES): - self.assertEqual( - i + 1, len(self.db._get_transaction_log())) - doc.content['ops'] += 1 - self.db.put_doc(doc) - docs.append(doc.doc_id) - - # assert length of transaction_log - transaction_log = self.db._get_transaction_log() - self.assertEqual( - REPEAT_TIMES + 1, len(transaction_log)) - - # assert that all entries in the log belong to the same doc - self.assertEqual(REPEAT_TIMES + 1, len(docs)) - for doc_id in docs: - self.assertEqual( - REPEAT_TIMES + 1, - len(filter(lambda t: t[0] == doc_id, transaction_log))) - - def test_correct_transaction_log_after_sequential_deletes(self): - """ - Assert that the transaction_log increases accordingly with sequential - puts and deletes. - """ - docs = [] - for i in range(0, REPEAT_TIMES): - doc = self.db.create_doc({'ops': 0}) - self.assertEqual( - 2 * i + 1, len(self.db._get_transaction_log())) - docs.append(doc.doc_id) - self.db.delete_doc(doc) - self.assertEqual( - 2 * i + 2, len(self.db._get_transaction_log())) - - # assert length of transaction_log - transaction_log = self.db._get_transaction_log() - self.assertEqual( - 2 * REPEAT_TIMES, len(transaction_log)) - - # assert that each doc appears twice in the transaction_log - self.assertEqual(REPEAT_TIMES, len(docs)) - for doc_id in docs: - self.assertEqual( - 2, - len(filter(lambda t: t[0] == doc_id, transaction_log))) - - @defer.inlineCallbacks - def test_correct_sync_log_after_sequential_syncs(self): - """ - Assert that the sync_log increases accordingly with sequential syncs. - """ - sol = self._soledad_instance( - auth_token='auth-token', - server_url=self.getURL()) - source_replica_uid = sol._dbpool.replica_uid - - def _create_docs(): - deferreds = [] - for i in xrange(0, REPEAT_TIMES): - deferreds.append(sol.create_doc({})) - return defer.gatherResults(deferreds) - - def _assert_transaction_and_sync_logs(results, sync_idx): - # assert sizes of transaction and sync logs - self.assertEqual( - sync_idx * REPEAT_TIMES, - len(self.db._get_transaction_log())) - gen, _ = self.db._get_replica_gen_and_trans_id(source_replica_uid) - self.assertEqual(sync_idx * REPEAT_TIMES, gen) - - def _assert_sync(results, sync_idx): - gen, docs = results - self.assertEqual((sync_idx + 1) * REPEAT_TIMES, gen) - self.assertEqual((sync_idx + 1) * REPEAT_TIMES, len(docs)) - # assert sizes of transaction and sync logs - self.assertEqual((sync_idx + 1) * REPEAT_TIMES, - len(self.db._get_transaction_log())) - target_known_gen, target_known_trans_id = \ - self.db._get_replica_gen_and_trans_id(source_replica_uid) - # assert it has the correct gen and trans_id - conn_key = sol._dbpool._u1dbconnections.keys().pop() - conn = sol._dbpool._u1dbconnections[conn_key] - sol_gen, sol_trans_id = conn._get_generation_info() - self.assertEqual(sol_gen, target_known_gen) - self.assertEqual(sol_trans_id, target_known_trans_id) - - # sync first time and assert success - results = yield _create_docs() - _assert_transaction_and_sync_logs(results, 0) - yield sol.sync() - results = yield sol.get_all_docs() - _assert_sync(results, 0) - - # create more docs, sync second time and assert success - results = yield _create_docs() - _assert_transaction_and_sync_logs(results, 1) - yield sol.sync() - results = yield sol.get_all_docs() - _assert_sync(results, 1) - - # - # Concurrency tests - # - - class _WorkerThread(threading.Thread): - - def __init__(self, params, run_method): - threading.Thread.__init__(self) - self._params = params - self._run_method = run_method - - def run(self): - self._run_method(self) - - def test_correct_transaction_log_after_concurrent_puts(self): - """ - Assert that the transaction_log increases accordingly with concurrent - puts. - """ - pool = threading.BoundedSemaphore(value=1) - threads = [] - docs = [] - - def _run_method(self): - doc = self._params['db'].create_doc({}) - pool.acquire() - self._params['docs'].append(doc.doc_id) - pool.release() - - for i in range(0, REPEAT_TIMES): - thread = self._WorkerThread( - {'docs': docs, 'db': self.db}, - _run_method) - thread.start() - threads.append(thread) - - for thread in threads: - thread.join() - - # assert length of transaction_log - transaction_log = self.db._get_transaction_log() - self.assertEqual( - REPEAT_TIMES, len(transaction_log)) - - # assert all documents are in the log - self.assertEqual(REPEAT_TIMES, len(docs)) - for doc_id in docs: - self.assertEqual( - 1, - len(filter(lambda t: t[0] == doc_id, transaction_log))) - - def test_correct_transaction_log_after_concurrent_deletes(self): - """ - Assert that the transaction_log increases accordingly with concurrent - puts and deletes. - """ - threads = [] - docs = [] - pool = threading.BoundedSemaphore(value=1) - - # create/delete method that will be run concurrently - def _run_method(self): - doc = self._params['db'].create_doc({}) - pool.acquire() - docs.append(doc.doc_id) - pool.release() - self._params['db'].delete_doc(doc) - - # launch concurrent threads - for i in range(0, REPEAT_TIMES): - thread = self._WorkerThread({'db': self.db}, _run_method) - thread.start() - threads.append(thread) - - # wait for threads to finish - for thread in threads: - thread.join() - - # assert transaction log - transaction_log = self.db._get_transaction_log() - self.assertEqual( - 2 * REPEAT_TIMES, len(transaction_log)) - # assert that each doc appears twice in the transaction_log - self.assertEqual(REPEAT_TIMES, len(docs)) - for doc_id in docs: - self.assertEqual( - 2, - len(filter(lambda t: t[0] == doc_id, transaction_log))) - - def test_correct_sync_log_after_concurrent_puts_and_sync(self): - """ - Assert that the sync_log is correct after concurrent syncs. - """ - docs = [] - - sol = self._soledad_instance( - auth_token='auth-token', - server_url=self.getURL()) - - def _save_doc_ids(results): - for doc in results: - docs.append(doc.doc_id) - - # create documents in parallel - deferreds = [] - for i in range(0, REPEAT_TIMES): - d = sol.create_doc({}) - deferreds.append(d) - - # wait for documents creation and sync - d = defer.gatherResults(deferreds) - d.addCallback(_save_doc_ids) - d.addCallback(lambda _: sol.sync()) - - def _assert_logs(results): - transaction_log = self.db._get_transaction_log() - self.assertEqual(REPEAT_TIMES, len(transaction_log)) - # assert all documents are in the remote log - self.assertEqual(REPEAT_TIMES, len(docs)) - for doc_id in docs: - self.assertEqual( - 1, - len(filter(lambda t: t[0] == doc_id, transaction_log))) - - d.addCallback(_assert_logs) - d.addCallback(lambda _: sol.close()) - - return d - - @defer.inlineCallbacks - def test_concurrent_syncs_do_not_fail(self): - """ - Assert that concurrent attempts to sync end up being executed - sequentially and do not fail. - """ - docs = [] - - sol = self._soledad_instance( - auth_token='auth-token', - server_url=self.getURL()) - - deferreds = [] - for i in xrange(0, REPEAT_TIMES): - d = sol.create_doc({}) - d.addCallback(lambda doc: docs.append(doc.doc_id)) - d.addCallback(lambda _: sol.sync()) - deferreds.append(d) - yield defer.gatherResults(deferreds, consumeErrors=True) - - transaction_log = self.db._get_transaction_log() - self.assertEqual(REPEAT_TIMES, len(transaction_log)) - # assert all documents are in the remote log - self.assertEqual(REPEAT_TIMES, len(docs)) - for doc_id in docs: - self.assertEqual( - 1, - len(filter(lambda t: t[0] == doc_id, transaction_log))) diff --git a/common/src/leap/soledad/common/tests/test_crypto.py b/common/src/leap/soledad/common/tests/test_crypto.py deleted file mode 100644 index 5ced024b..00000000 --- a/common/src/leap/soledad/common/tests/test_crypto.py +++ /dev/null @@ -1,226 +0,0 @@ -# -*- coding: utf-8 -*- -# test_crypto.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Tests for cryptographic related stuff. -""" -import os -import hashlib -import binascii - -from leap.soledad.client import crypto -from leap.soledad.common.document import SoledadDocument -from leap.soledad.common.tests.util import BaseSoledadTest -from leap.soledad.common.crypto import WrongMacError -from leap.soledad.common.crypto import UnknownMacMethodError -from leap.soledad.common.crypto import ENC_JSON_KEY -from leap.soledad.common.crypto import ENC_SCHEME_KEY -from leap.soledad.common.crypto import MAC_KEY -from leap.soledad.common.crypto import MAC_METHOD_KEY - - -class EncryptedSyncTestCase(BaseSoledadTest): - - """ - Tests that guarantee that data will always be encrypted when syncing. - """ - - def test_encrypt_decrypt_json(self): - """ - Test encrypting and decrypting documents. - """ - simpledoc = {'key': 'val'} - doc1 = SoledadDocument(doc_id='id') - doc1.content = simpledoc - - # encrypt doc - doc1.set_json(self._soledad._crypto.encrypt_doc(doc1)) - # assert content is different and includes keys - self.assertNotEqual( - simpledoc, doc1.content, - 'incorrect document encryption') - self.assertTrue(ENC_JSON_KEY in doc1.content) - self.assertTrue(ENC_SCHEME_KEY in doc1.content) - # decrypt doc - doc1.set_json(self._soledad._crypto.decrypt_doc(doc1)) - self.assertEqual( - simpledoc, doc1.content, 'incorrect document encryption') - - -class RecoveryDocumentTestCase(BaseSoledadTest): - - def test_export_recovery_document_raw(self): - rd = self._soledad.secrets._export_recovery_document() - secret_id = rd[self._soledad.secrets.STORAGE_SECRETS_KEY].items()[0][0] - # assert exported secret is the same - secret = self._soledad.secrets._decrypt_storage_secret_version_1( - rd[self._soledad.secrets.STORAGE_SECRETS_KEY][secret_id]) - self.assertEqual(secret_id, self._soledad.secrets._secret_id) - self.assertEqual(secret, self._soledad.secrets._secrets[secret_id]) - # assert recovery document structure - encrypted_secret = rd[ - self._soledad.secrets.STORAGE_SECRETS_KEY][secret_id] - self.assertTrue(self._soledad.secrets.CIPHER_KEY in encrypted_secret) - self.assertTrue( - encrypted_secret[self._soledad.secrets.CIPHER_KEY] == 'aes256') - self.assertTrue(self._soledad.secrets.LENGTH_KEY in encrypted_secret) - self.assertTrue(self._soledad.secrets.SECRET_KEY in encrypted_secret) - - def test_import_recovery_document(self): - rd = self._soledad.secrets._export_recovery_document() - s = self._soledad_instance() - s.secrets._import_recovery_document(rd) - s.secrets.set_secret_id(self._soledad.secrets._secret_id) - self.assertEqual(self._soledad.storage_secret, - s.storage_secret, - 'Failed settinng secret for symmetric encryption.') - s.close() - - -class SoledadSecretsTestCase(BaseSoledadTest): - - def test_new_soledad_instance_generates_one_secret(self): - self.assertTrue( - self._soledad.storage_secret is not None, - "Expected secret to be something different than None") - number_of_secrets = len(self._soledad.secrets._secrets) - self.assertTrue( - number_of_secrets == 1, - "Expected exactly 1 secret, got %d instead." % number_of_secrets) - - def test_generated_secret_is_of_correct_type(self): - expected_type = str - self.assertIsInstance( - self._soledad.storage_secret, expected_type, - "Expected secret to be of type %s" % expected_type) - - def test_generated_secret_has_correct_lengt(self): - expected_length = self._soledad.secrets.GEN_SECRET_LENGTH - actual_length = len(self._soledad.storage_secret) - self.assertTrue( - expected_length == actual_length, - "Expected secret with length %d, got %d instead." - % (expected_length, actual_length)) - - def test_generated_secret_id_is_sha256_hash_of_secret(self): - generated = self._soledad.secrets.secret_id - expected = hashlib.sha256(self._soledad.storage_secret).hexdigest() - self.assertTrue( - generated == expected, - "Expeceted generated secret id to be sha256 hash, got something " - "else instead.") - - def test_generate_new_secret_generates_different_secret_id(self): - # generate new secret - secret_id_1 = self._soledad.secrets.secret_id - secret_id_2 = self._soledad.secrets._gen_secret() - self.assertTrue( - len(self._soledad.secrets._secrets) == 2, - "Expected exactly 2 secrets.") - self.assertTrue( - secret_id_1 != secret_id_2, - "Expected IDs of secrets to be distinct.") - self.assertTrue( - secret_id_1 in self._soledad.secrets._secrets, - "Expected to find ID of first secret in Soledad Secrets.") - self.assertTrue( - secret_id_2 in self._soledad.secrets._secrets, - "Expected to find ID of second secret in Soledad Secrets.") - - def test__has_secret(self): - self.assertTrue( - self._soledad._secrets._has_secret(), - "Should have a secret at this point") - - -class MacAuthTestCase(BaseSoledadTest): - - def test_decrypt_with_wrong_mac_raises(self): - """ - Trying to decrypt a document with wrong MAC should raise. - """ - simpledoc = {'key': 'val'} - doc = SoledadDocument(doc_id='id') - doc.content = simpledoc - # encrypt doc - doc.set_json(self._soledad._crypto.encrypt_doc(doc)) - self.assertTrue(MAC_KEY in doc.content) - self.assertTrue(MAC_METHOD_KEY in doc.content) - # mess with MAC - doc.content[MAC_KEY] = '1234567890ABCDEF' - # try to decrypt doc - self.assertRaises( - WrongMacError, - self._soledad._crypto.decrypt_doc, doc) - - def test_decrypt_with_unknown_mac_method_raises(self): - """ - Trying to decrypt a document with unknown MAC method should raise. - """ - simpledoc = {'key': 'val'} - doc = SoledadDocument(doc_id='id') - doc.content = simpledoc - # encrypt doc - doc.set_json(self._soledad._crypto.encrypt_doc(doc)) - self.assertTrue(MAC_KEY in doc.content) - self.assertTrue(MAC_METHOD_KEY in doc.content) - # mess with MAC method - doc.content[MAC_METHOD_KEY] = 'mymac' - # try to decrypt doc - self.assertRaises( - UnknownMacMethodError, - self._soledad._crypto.decrypt_doc, doc) - - -class SoledadCryptoAESTestCase(BaseSoledadTest): - - def test_encrypt_decrypt_sym(self): - # generate 256-bit key - key = os.urandom(32) - iv, cyphertext = crypto.encrypt_sym('data', key) - self.assertTrue(cyphertext is not None) - self.assertTrue(cyphertext != '') - self.assertTrue(cyphertext != 'data') - plaintext = crypto.decrypt_sym(cyphertext, key, iv) - self.assertEqual('data', plaintext) - - def test_decrypt_with_wrong_iv_fails(self): - key = os.urandom(32) - iv, cyphertext = crypto.encrypt_sym('data', key) - self.assertTrue(cyphertext is not None) - self.assertTrue(cyphertext != '') - self.assertTrue(cyphertext != 'data') - # get a different iv by changing the first byte - rawiv = binascii.a2b_base64(iv) - wrongiv = rawiv - while wrongiv == rawiv: - wrongiv = os.urandom(1) + rawiv[1:] - plaintext = crypto.decrypt_sym( - cyphertext, key, iv=binascii.b2a_base64(wrongiv)) - self.assertNotEqual('data', plaintext) - - def test_decrypt_with_wrong_key_fails(self): - key = os.urandom(32) - iv, cyphertext = crypto.encrypt_sym('data', key) - self.assertTrue(cyphertext is not None) - self.assertTrue(cyphertext != '') - self.assertTrue(cyphertext != 'data') - wrongkey = os.urandom(32) # 256-bits key - # ensure keys are different in case we are extremely lucky - while wrongkey == key: - wrongkey = os.urandom(32) - plaintext = crypto.decrypt_sym(cyphertext, wrongkey, iv) - self.assertNotEqual('data', plaintext) diff --git a/common/src/leap/soledad/common/tests/test_encdecpool.py b/common/src/leap/soledad/common/tests/test_encdecpool.py deleted file mode 100644 index c626561d..00000000 --- a/common/src/leap/soledad/common/tests/test_encdecpool.py +++ /dev/null @@ -1,315 +0,0 @@ -# -*- coding: utf-8 -*- -# test_encdecpool.py -# Copyright (C) 2015 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Tests for encryption and decryption pool. -""" -import json -from random import shuffle - -from mock import MagicMock -from twisted.internet.defer import inlineCallbacks - -from leap.soledad.client.encdecpool import SyncEncrypterPool -from leap.soledad.client.encdecpool import SyncDecrypterPool - -from leap.soledad.common.document import SoledadDocument -from leap.soledad.common.tests.util import BaseSoledadTest -from twisted.internet import defer -from twisted.test.proto_helpers import MemoryReactorClock - -DOC_ID = "mydoc" -DOC_REV = "rev" -DOC_CONTENT = {'simple': 'document'} - - -class TestSyncEncrypterPool(BaseSoledadTest): - - def setUp(self): - BaseSoledadTest.setUp(self) - crypto = self._soledad._crypto - sync_db = self._soledad._sync_db - self._pool = SyncEncrypterPool(crypto, sync_db) - self._pool.start() - - def tearDown(self): - self._pool.stop() - BaseSoledadTest.tearDown(self) - - @inlineCallbacks - def test_get_encrypted_doc_returns_none(self): - """ - Test that trying to get an encrypted doc from the pool returns None if - the document was never added for encryption. - """ - doc = yield self._pool.get_encrypted_doc(DOC_ID, DOC_REV) - self.assertIsNone(doc) - - @inlineCallbacks - def test_encrypt_doc_and_get_it_back(self): - """ - Test that the pool actually encrypts a document added to the queue. - """ - doc = SoledadDocument( - doc_id=DOC_ID, rev=DOC_REV, json=json.dumps(DOC_CONTENT)) - self._pool.encrypt_doc(doc) - - # exhaustivelly attempt to get the encrypted document - encrypted = None - attempts = 0 - while encrypted is None and attempts < 10: - encrypted = yield self._pool.get_encrypted_doc(DOC_ID, DOC_REV) - attempts += 1 - - self.assertIsNotNone(encrypted) - self.assertTrue(attempts < 10) - - -class TestSyncDecrypterPool(BaseSoledadTest): - - def _insert_doc_cb(self, doc, gen, trans_id): - """ - Method used to mock the sync's return_doc_cb callback. - """ - self._inserted_docs.append((doc, gen, trans_id)) - - def _setup_pool(self, sync_db=None): - sync_db = sync_db or self._soledad._sync_db - return SyncDecrypterPool( - self._soledad._crypto, - sync_db, - source_replica_uid=self._soledad._dbpool.replica_uid, - insert_doc_cb=self._insert_doc_cb) - - def setUp(self): - BaseSoledadTest.setUp(self) - # setup the pool - self._pool = self._setup_pool() - # reset the inserted docs mock - self._inserted_docs = [] - - def tearDown(self): - if self._pool.running: - self._pool.stop() - BaseSoledadTest.tearDown(self) - - def test_insert_received_doc(self): - """ - Test that one document added to the pool is inserted using the - callback. - """ - self._pool.start(1) - self._pool.insert_received_doc( - DOC_ID, DOC_REV, "{}", 1, "trans_id", 1) - - def _assert_doc_was_inserted(_): - self.assertEqual( - self._inserted_docs, - [(SoledadDocument(DOC_ID, DOC_REV, "{}"), 1, u"trans_id")]) - - self._pool.deferred.addCallback(_assert_doc_was_inserted) - return self._pool.deferred - - def test_looping_control(self): - """ - Start and stop cleanly. - """ - self._pool.start(10) - self.assertTrue(self._pool.running) - self._pool.stop() - self.assertFalse(self._pool.running) - self.assertTrue(self._pool.deferred.called) - - def test_sync_id_col_is_created_if_non_existing_in_docs_recvd_table(self): - """ - Test that docs_received table is migrated, and has the sync_id column - """ - mock_run_query = MagicMock(return_value=defer.succeed(None)) - mock_sync_db = MagicMock() - mock_sync_db.runQuery = mock_run_query - pool = self._setup_pool(mock_sync_db) - d = pool.start(10) - pool.stop() - - def assert_trial_to_create_sync_id_column(_): - mock_run_query.assert_called_once_with( - "ALTER TABLE docs_received ADD COLUMN sync_id") - - d.addCallback(assert_trial_to_create_sync_id_column) - return d - - def test_insert_received_doc_many(self): - """ - Test that many documents added to the pool are inserted using the - callback. - """ - many = 100 - self._pool.start(many) - - # insert many docs in the pool - for i in xrange(many): - gen = idx = i + 1 - doc_id = "doc_id: %d" % idx - rev = "rev: %d" % idx - content = {'idx': idx} - trans_id = "trans_id: %d" % idx - self._pool.insert_received_doc( - doc_id, rev, content, gen, trans_id, idx) - - def _assert_doc_was_inserted(_): - self.assertEqual(many, len(self._inserted_docs)) - idx = 1 - for doc, gen, trans_id in self._inserted_docs: - expected_gen = idx - expected_doc_id = "doc_id: %d" % idx - expected_rev = "rev: %d" % idx - expected_content = json.dumps({'idx': idx}) - expected_trans_id = "trans_id: %d" % idx - - self.assertEqual(expected_doc_id, doc.doc_id) - self.assertEqual(expected_rev, doc.rev) - self.assertEqual(expected_content, json.dumps(doc.content)) - self.assertEqual(expected_gen, gen) - self.assertEqual(expected_trans_id, trans_id) - - idx += 1 - - self._pool.deferred.addCallback(_assert_doc_was_inserted) - return self._pool.deferred - - def test_insert_encrypted_received_doc(self): - """ - Test that one encrypted document added to the pool is decrypted and - inserted using the callback. - """ - crypto = self._soledad._crypto - doc = SoledadDocument( - doc_id=DOC_ID, rev=DOC_REV, json=json.dumps(DOC_CONTENT)) - encrypted_content = json.loads(crypto.encrypt_doc(doc)) - - # insert the encrypted document in the pool - self._pool.start(1) - self._pool.insert_encrypted_received_doc( - DOC_ID, DOC_REV, encrypted_content, 1, "trans_id", 1) - - def _assert_doc_was_decrypted_and_inserted(_): - self.assertEqual(1, len(self._inserted_docs)) - self.assertEqual(self._inserted_docs, [(doc, 1, u"trans_id")]) - - self._pool.deferred.addCallback( - _assert_doc_was_decrypted_and_inserted) - return self._pool.deferred - - @inlineCallbacks - def test_processing_order(self): - """ - This test ensures that processing of documents only occur if there is - a sequence in place. - """ - reactor_clock = MemoryReactorClock() - self._pool._loop.clock = reactor_clock - - crypto = self._soledad._crypto - - docs = [] - for i in xrange(1, 10): - i = str(i) - doc = SoledadDocument( - doc_id=DOC_ID + i, rev=DOC_REV + i, - json=json.dumps(DOC_CONTENT)) - encrypted_content = json.loads(crypto.encrypt_doc(doc)) - docs.append((doc, encrypted_content)) - - # insert the encrypted document in the pool - self._pool.start(10) # pool is expecting to process 10 docs - # first three arrives, forming a sequence - for i, (doc, encrypted_content) in enumerate(docs[:3]): - gen = idx = i + 1 - yield self._pool.insert_encrypted_received_doc( - doc.doc_id, doc.rev, encrypted_content, gen, "trans_id", idx) - # last one arrives alone, so it can't be processed - doc, encrypted_content = docs[-1] - yield self._pool.insert_encrypted_received_doc( - doc.doc_id, doc.rev, encrypted_content, 10, "trans_id", 10) - - reactor_clock.advance(self._pool.DECRYPT_LOOP_PERIOD) - yield self._pool._decrypt_and_recurse() - - self.assertEqual(3, self._pool._processed_docs) - - def test_insert_encrypted_received_doc_many(self, many=100): - """ - Test that many encrypted documents added to the pool are decrypted and - inserted using the callback. - """ - crypto = self._soledad._crypto - self._pool.start(many) - docs = [] - - # insert many encrypted docs in the pool - for i in xrange(many): - gen = idx = i + 1 - doc_id = "doc_id: %d" % idx - rev = "rev: %d" % idx - content = {'idx': idx} - trans_id = "trans_id: %d" % idx - - doc = SoledadDocument( - doc_id=doc_id, rev=rev, json=json.dumps(content)) - - encrypted_content = json.loads(crypto.encrypt_doc(doc)) - docs.append((doc_id, rev, encrypted_content, gen, - trans_id, idx)) - shuffle(docs) - - for doc in docs: - self._pool.insert_encrypted_received_doc(*doc) - - def _assert_docs_were_decrypted_and_inserted(_): - self.assertEqual(many, len(self._inserted_docs)) - idx = 1 - for doc, gen, trans_id in self._inserted_docs: - expected_gen = idx - expected_doc_id = "doc_id: %d" % idx - expected_rev = "rev: %d" % idx - expected_content = json.dumps({'idx': idx}) - expected_trans_id = "trans_id: %d" % idx - - self.assertEqual(expected_doc_id, doc.doc_id) - self.assertEqual(expected_rev, doc.rev) - self.assertEqual(expected_content, json.dumps(doc.content)) - self.assertEqual(expected_gen, gen) - self.assertEqual(expected_trans_id, trans_id) - - idx += 1 - - self._pool.deferred.addCallback( - _assert_docs_were_decrypted_and_inserted) - return self._pool.deferred - - @inlineCallbacks - def test_pool_reuse(self): - """ - The pool is reused between syncs, this test verifies that - reusing is fine. - """ - for i in xrange(3): - yield self.test_insert_encrypted_received_doc_many(5) - self._inserted_docs = [] - decrypted_docs = yield self._pool._get_docs(encrypted=False) - # check that decrypted docs staging is clean - self.assertEquals([], decrypted_docs) - self._pool.stop() diff --git a/common/src/leap/soledad/common/tests/test_http.py b/common/src/leap/soledad/common/tests/test_http.py deleted file mode 100644 index 2351748d..00000000 --- a/common/src/leap/soledad/common/tests/test_http.py +++ /dev/null @@ -1,60 +0,0 @@ -# -*- coding: utf-8 -*- -# test_http.py -# Copyright (C) 2013, 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Test Leap backend bits: test http database -""" - -from leap.soledad.client import auth -from leap.soledad.common.tests.u1db_tests import test_http_database -from leap.soledad.common.l2db.remote import http_database - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_http_database`. -# ----------------------------------------------------------------------------- - -class _HTTPDatabase(http_database.HTTPDatabase, auth.TokenBasedAuth): - - """ - Wraps our token auth implementation. - """ - - def set_token_credentials(self, uuid, token): - auth.TokenBasedAuth.set_token_credentials(self, uuid, token) - - def _sign_request(self, method, url_query, params): - return auth.TokenBasedAuth._sign_request( - self, method, url_query, params) - - -class TestHTTPDatabaseWithCreds( - test_http_database.TestHTTPDatabaseCtrWithCreds): - - def test_get_sync_target_inherits_token_credentials(self): - # this test was from TestDatabaseSimpleOperations but we put it here - # for convenience. - self.db = _HTTPDatabase('dbase') - self.db.set_token_credentials('user-uuid', 'auth-token') - st = self.db.get_sync_target() - self.assertEqual(self.db._creds, st._creds) - - def test_ctr_with_creds(self): - db1 = _HTTPDatabase('http://dbs/db', creds={'token': { - 'uuid': 'user-uuid', - 'token': 'auth-token', - }}) - self.assertIn('token', db1._creds) diff --git a/common/src/leap/soledad/common/tests/test_http_client.py b/common/src/leap/soledad/common/tests/test_http_client.py deleted file mode 100644 index d932b2b0..00000000 --- a/common/src/leap/soledad/common/tests/test_http_client.py +++ /dev/null @@ -1,117 +0,0 @@ -# -*- coding: utf-8 -*- -# test_http_client.py -# Copyright (C) 2013-2016 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Test Leap backend bits: sync target -""" -import json - -from testscenarios import TestWithScenarios - -from leap.soledad.client import auth -from leap.soledad.common.l2db.remote import http_client -from leap.soledad.common.tests.u1db_tests import test_http_client -from leap.soledad.server.auth import SoledadTokenAuthMiddleware - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_http_client`. -# ----------------------------------------------------------------------------- - -class TestSoledadClientBase( - TestWithScenarios, - test_http_client.TestHTTPClientBase): - - """ - This class should be used to test Token auth. - """ - - def getClientWithToken(self, **kwds): - self.startServer() - - class _HTTPClientWithToken( - http_client.HTTPClientBase, auth.TokenBasedAuth): - - def set_token_credentials(self, uuid, token): - auth.TokenBasedAuth.set_token_credentials(self, uuid, token) - - def _sign_request(self, method, url_query, params): - return auth.TokenBasedAuth._sign_request( - self, method, url_query, params) - - return _HTTPClientWithToken(self.getURL('dbase'), **kwds) - - def test_oauth(self): - """ - Suppress oauth test (we test for token auth here). - """ - pass - - def test_oauth_ctr_creds(self): - """ - Suppress oauth test (we test for token auth here). - """ - pass - - def test_oauth_Unauthorized(self): - """ - Suppress oauth test (we test for token auth here). - """ - pass - - def app(self, environ, start_response): - res = test_http_client.TestHTTPClientBase.app( - self, environ, start_response) - if res is not None: - return res - # mime solead application here. - if '/token' in environ['PATH_INFO']: - auth = environ.get(SoledadTokenAuthMiddleware.HTTP_AUTH_KEY) - if not auth: - start_response("401 Unauthorized", - [('Content-Type', 'application/json')]) - return [json.dumps({"error": "unauthorized", - "message": e.message})] - scheme, encoded = auth.split(None, 1) - if scheme.lower() != 'token': - start_response("401 Unauthorized", - [('Content-Type', 'application/json')]) - return [json.dumps({"error": "unauthorized", - "message": e.message})] - uuid, token = encoded.decode('base64').split(':', 1) - if uuid != 'user-uuid' and token != 'auth-token': - return Exception("Incorrect address or token.") - start_response("200 OK", [('Content-Type', 'application/json')]) - return [json.dumps([environ['PATH_INFO'], uuid, token])] - - def test_token(self): - """ - Test if token is sent correctly. - """ - cli = self.getClientWithToken() - cli.set_token_credentials('user-uuid', 'auth-token') - res, headers = cli._request('GET', ['doc', 'token']) - self.assertEqual( - ['/dbase/doc/token', 'user-uuid', 'auth-token'], json.loads(res)) - - def test_token_ctr_creds(self): - cli = self.getClientWithToken(creds={'token': { - 'uuid': 'user-uuid', - 'token': 'auth-token', - }}) - res, headers = cli._request('GET', ['doc', 'token']) - self.assertEqual( - ['/dbase/doc/token', 'user-uuid', 'auth-token'], json.loads(res)) diff --git a/common/src/leap/soledad/common/tests/test_https.py b/common/src/leap/soledad/common/tests/test_https.py deleted file mode 100644 index 8d9b8d92..00000000 --- a/common/src/leap/soledad/common/tests/test_https.py +++ /dev/null @@ -1,137 +0,0 @@ -# -*- coding: utf-8 -*- -# test_sync_target.py -# Copyright (C) 2013, 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Test Leap backend bits: https -""" -from unittest import skip - -from testscenarios import TestWithScenarios - -from leap.soledad import client - -from leap.soledad.common.l2db.remote import http_client -from leap.soledad.common.tests.u1db_tests import test_backends -from leap.soledad.common.tests.u1db_tests import test_https -from leap.soledad.common.tests.util import ( - BaseSoledadTest, - make_soledad_document_for_test, - make_soledad_app, - make_token_soledad_app, -) - - -LEAP_SCENARIOS = [ - ('http', { - 'make_database_for_test': test_backends.make_http_database_for_test, - 'copy_database_for_test': test_backends.copy_http_database_for_test, - 'make_document_for_test': make_soledad_document_for_test, - 'make_app_with_state': make_soledad_app}), -] - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_https`. -# ----------------------------------------------------------------------------- - -def token_leap_https_sync_target(test, host, path, cert_file=None): - _, port = test.server.server_address - # source_replica_uid = test._soledad._dbpool.replica_uid - creds = {'token': {'uuid': 'user-uuid', 'token': 'auth-token'}} - if not cert_file: - cert_file = test.cacert_pem - st = client.http_target.SoledadHTTPSyncTarget( - 'https://%s:%d/%s' % (host, port, path), - source_replica_uid='other-id', - creds=creds, - crypto=test._soledad._crypto, - cert_file=cert_file) - return st - - -@skip("Skiping tests imported from U1DB.") -class TestSoledadHTTPSyncTargetHttpsSupport( - TestWithScenarios, - # test_https.TestHttpSyncTargetHttpsSupport, - BaseSoledadTest): - - scenarios = [ - ('token_soledad_https', - { - #'server_def': test_https.https_server_def, - 'make_app_with_state': make_token_soledad_app, - 'make_document_for_test': make_soledad_document_for_test, - 'sync_target': token_leap_https_sync_target}), - ] - - def setUp(self): - # the parent constructor undoes our SSL monkey patch to ensure tests - # run smoothly with standard u1db. - test_https.TestHttpSyncTargetHttpsSupport.setUp(self) - # so here monkey patch again to test our functionality. - api = client.api - http_client._VerifiedHTTPSConnection = api.VerifiedHTTPSConnection - client.api.SOLEDAD_CERT = http_client.CA_CERTS - - def test_cannot_verify_cert(self): - self.startServer() - # don't print expected traceback server-side - self.server.handle_error = lambda req, cli_addr: None - self.request_state._create_database('test') - remote_target = self.getSyncTarget( - 'localhost', 'test', cert_file=http_client.CA_CERTS) - d = remote_target.record_sync_info('other-id', 2, 'T-id') - - def _assert_raises(result): - from twisted.python.failure import Failure - if isinstance(result, Failure): - from OpenSSL.SSL import Error - error = result.value.message[0].value - if isinstance(error, Error): - msg = error.message[0][2] - self.assertEqual("certificate verify failed", msg) - return - self.fail("certificate verification should have failed.") - - d.addCallbacks(_assert_raises, _assert_raises) - return d - - def test_working(self): - """ - Test that SSL connections work well. - - This test was adapted to patch Soledad's HTTPS connection custom class - with the intended CA certificates. - """ - self.startServer() - db = self.request_state._create_database('test') - remote_target = self.getSyncTarget('localhost', 'test') - d = remote_target.record_sync_info('other-id', 2, 'T-id') - d.addCallback(lambda _: - self.assertEqual( - (2, 'T-id'), - db._get_replica_gen_and_trans_id('other-id') - )) - d.addCallback(lambda _: remote_target.close()) - return d - - def test_host_mismatch(self): - """ - This test is disabled because soledad's twisted-based http agent uses - pyOpenSSL, which will complain if we try to use an IP to connect to - the remote host (see the original test in u1db_tests/test_https.py). - """ - pass diff --git a/common/src/leap/soledad/common/tests/test_server.py b/common/src/leap/soledad/common/tests/test_server.py deleted file mode 100644 index 357027e9..00000000 --- a/common/src/leap/soledad/common/tests/test_server.py +++ /dev/null @@ -1,537 +0,0 @@ -# -*- coding: utf-8 -*- -# test_server.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Tests for server-related functionality. -""" -import binascii -import mock -import os -import tempfile -import time - -from hashlib import sha512 -from pkg_resources import resource_filename -from urlparse import urljoin -from uuid import uuid4 - -from twisted.internet import defer -from twisted.trial import unittest - -from leap.soledad.common.couch.state import CouchServerState -from leap.soledad.common.couch import CouchDatabase -from leap.soledad.common.tests.u1db_tests import TestCaseWithServer -from leap.soledad.common.tests.test_couch import CouchDBTestCase -from leap.soledad.common.tests.util import ( - make_token_soledad_app, - make_soledad_document_for_test, - soledad_sync_target, - BaseSoledadTest, -) - -from leap.soledad.common import crypto -from leap.soledad.client import Soledad -from leap.soledad.server import load_configuration -from leap.soledad.server import CONFIG_DEFAULTS -from leap.soledad.server.auth import URLToAuthorization -from leap.soledad.server.auth import SoledadTokenAuthMiddleware - - -class ServerAuthenticationMiddlewareTestCase(CouchDBTestCase): - - def setUp(self): - super(ServerAuthenticationMiddlewareTestCase, self).setUp() - app = mock.Mock() - self._state = CouchServerState(self.couch_url) - app.state = self._state - self.auth_middleware = SoledadTokenAuthMiddleware(app) - self._authorize('valid-uuid', 'valid-token') - - def _authorize(self, uuid, token): - token_doc = {} - token_doc['_id'] = sha512(token).hexdigest() - token_doc[self._state.TOKENS_USER_ID_KEY] = uuid - token_doc[self._state.TOKENS_TYPE_KEY] = \ - self._state.TOKENS_TYPE_DEF - dbname = self._state._tokens_dbname() - db = self.couch_server.create(dbname) - db.save(token_doc) - self.addCleanup(self.delete_db, db.name) - - def test_authorized_user(self): - is_authorized = self.auth_middleware._verify_authentication_data - self.assertTrue(is_authorized('valid-uuid', 'valid-token')) - self.assertFalse(is_authorized('valid-uuid', 'invalid-token')) - self.assertFalse(is_authorized('invalid-uuid', 'valid-token')) - self.assertFalse(is_authorized('eve', 'invalid-token')) - - -class ServerAuthorizationTestCase(BaseSoledadTest): - - """ - Tests related to Soledad server authorization. - """ - - def setUp(self): - pass - - def tearDown(self): - pass - - def _make_environ(self, path_info, request_method): - return { - 'PATH_INFO': path_info, - 'REQUEST_METHOD': request_method, - } - - def test_verify_action_with_correct_dbnames(self): - """ - Test encrypting and decrypting documents. - - The following table lists the authorized actions among all possible - u1db remote actions: - - URL path | Authorized actions - -------------------------------------------------- - / | GET - /shared-db | GET - /shared-db/docs | - - /shared-db/doc/{id} | GET, PUT, DELETE - /shared-db/sync-from/{source} | - - /user-db | GET, PUT, DELETE - /user-db/docs | - - /user-db/doc/{id} | - - /user-db/sync-from/{source} | GET, PUT, POST - """ - uuid = uuid4().hex - authmap = URLToAuthorization(uuid,) - dbname = authmap._user_db_name - # test global auth - self.assertTrue( - authmap.is_authorized(self._make_environ('/', 'GET'))) - # test shared-db database resource auth - self.assertTrue( - authmap.is_authorized( - self._make_environ('/shared', 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared', 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared', 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared', 'POST'))) - # test shared-db docs resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/docs', 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/docs', 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/docs', 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/docs', 'POST'))) - # test shared-db doc resource auth - self.assertTrue( - authmap.is_authorized( - self._make_environ('/shared/doc/x', 'GET'))) - self.assertTrue( - authmap.is_authorized( - self._make_environ('/shared/doc/x', 'PUT'))) - self.assertTrue( - authmap.is_authorized( - self._make_environ('/shared/doc/x', 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/doc/x', 'POST'))) - # test shared-db sync resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/sync-from/x', 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/sync-from/x', 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/sync-from/x', 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/shared/sync-from/x', 'POST'))) - # test user-db database resource auth - self.assertTrue( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'GET'))) - self.assertTrue( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'PUT'))) - self.assertTrue( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'POST'))) - # test user-db docs resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'POST'))) - # test user-db doc resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'POST'))) - # test user-db sync resource auth - self.assertTrue( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'GET'))) - self.assertTrue( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'DELETE'))) - self.assertTrue( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'POST'))) - - def test_verify_action_with_wrong_dbnames(self): - """ - Test if authorization fails for a wrong dbname. - """ - uuid = uuid4().hex - authmap = URLToAuthorization(uuid) - dbname = 'somedb' - # test wrong-db database resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s' % dbname, 'POST'))) - # test wrong-db docs resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/docs' % dbname, 'POST'))) - # test wrong-db doc resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/doc/x' % dbname, 'POST'))) - # test wrong-db sync resource auth - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'GET'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'PUT'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'DELETE'))) - self.assertFalse( - authmap.is_authorized( - self._make_environ('/%s/sync-from/x' % dbname, 'POST'))) - - -class EncryptedSyncTestCase( - CouchDBTestCase, TestCaseWithServer): - - """ - Tests for encrypted sync using Soledad server backed by a couch database. - """ - - # increase twisted.trial's timeout because large files syncing might take - # some time to finish. - timeout = 500 - - @staticmethod - def make_app_with_state(state): - return make_token_soledad_app(state) - - make_document_for_test = make_soledad_document_for_test - - sync_target = soledad_sync_target - - def _soledad_instance(self, user=None, passphrase=u'123', - prefix='', - secrets_path='secrets.json', - local_db_path='soledad.u1db', - server_url='', - cert_file=None, auth_token=None): - """ - Instantiate Soledad. - """ - - # this callback ensures we save a document which is sent to the shared - # db. - def _put_doc_side_effect(doc): - self._doc_put = doc - - if not server_url: - # attempt to find the soledad server url - server_address = None - server = getattr(self, 'server', None) - if server: - server_address = getattr(self.server, 'server_address', None) - else: - host = self.port.getHost() - server_address = (host.host, host.port) - if server_address: - server_url = 'http://%s:%d' % (server_address) - - return Soledad( - user, - passphrase, - secrets_path=os.path.join(self.tempdir, prefix, secrets_path), - local_db_path=os.path.join( - self.tempdir, prefix, local_db_path), - server_url=server_url, - cert_file=cert_file, - auth_token=auth_token, - shared_db=self.get_default_shared_mock(_put_doc_side_effect)) - - def make_app(self): - self.request_state = CouchServerState(self.couch_url) - return self.make_app_with_state(self.request_state) - - def setUp(self): - # the order of the following initializations is crucial because of - # dependencies. - # XXX explain better - CouchDBTestCase.setUp(self) - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - TestCaseWithServer.setUp(self) - - def tearDown(self): - CouchDBTestCase.tearDown(self) - TestCaseWithServer.tearDown(self) - - def _test_encrypted_sym_sync(self, passphrase=u'123', doc_size=2, - number_of_docs=1): - """ - Test the complete syncing chain between two soledad dbs using a - Soledad server backed by a couch database. - """ - self.startTwistedServer() - user = 'user-' + uuid4().hex - - # this will store all docs ids to avoid get_all_docs - created_ids = [] - - # instantiate soledad and create a document - sol1 = self._soledad_instance( - user=user, - # token is verified in test_target.make_token_soledad_app - auth_token='auth-token', - passphrase=passphrase) - - # instantiate another soledad using the same secret as the previous - # one (so we can correctly verify the mac of the synced document) - sol2 = self._soledad_instance( - user=user, - prefix='x', - auth_token='auth-token', - secrets_path=sol1._secrets_path, - passphrase=passphrase) - - # ensure remote db exists before syncing - db = CouchDatabase.open_database( - urljoin(self.couch_url, 'user-' + user), - create=True, - ensure_ddocs=True) - - def _db1AssertEmptyDocList(results): - _, doclist = results - self.assertEqual([], doclist) - - def _db1CreateDocs(results): - deferreds = [] - for i in xrange(number_of_docs): - content = binascii.hexlify(os.urandom(doc_size / 2)) - d = sol1.create_doc({'data': content}) - d.addCallback(created_ids.append) - deferreds.append(d) - return defer.DeferredList(deferreds) - - def _db1AssertDocsSyncedToServer(results): - self.assertEqual(number_of_docs, len(created_ids)) - for soldoc in created_ids: - couchdoc = db.get_doc(soldoc.doc_id) - self.assertTrue(couchdoc) - # assert document structure in couch server - self.assertEqual(soldoc.doc_id, couchdoc.doc_id) - self.assertEqual(soldoc.rev, couchdoc.rev) - couch_content = couchdoc.content.keys() - self.assertEqual(6, len(couch_content)) - self.assertTrue(crypto.ENC_JSON_KEY in couch_content) - self.assertTrue(crypto.ENC_SCHEME_KEY in couch_content) - self.assertTrue(crypto.ENC_METHOD_KEY in couch_content) - self.assertTrue(crypto.ENC_IV_KEY in couch_content) - self.assertTrue(crypto.MAC_KEY in couch_content) - self.assertTrue(crypto.MAC_METHOD_KEY in couch_content) - - d = sol1.get_all_docs() - d.addCallback(_db1AssertEmptyDocList) - d.addCallback(_db1CreateDocs) - d.addCallback(lambda _: sol1.sync()) - d.addCallback(_db1AssertDocsSyncedToServer) - - def _db2AssertEmptyDocList(results): - _, doclist = results - self.assertEqual([], doclist) - - def _getAllDocsFromBothDbs(results): - d1 = sol1.get_all_docs() - d2 = sol2.get_all_docs() - return defer.DeferredList([d1, d2]) - - d.addCallback(lambda _: sol2.get_all_docs()) - d.addCallback(_db2AssertEmptyDocList) - d.addCallback(lambda _: sol2.sync()) - d.addCallback(_getAllDocsFromBothDbs) - - def _assertDocSyncedFromDb1ToDb2(results): - r1, r2 = results - _, (gen1, doclist1) = r1 - _, (gen2, doclist2) = r2 - self.assertEqual(number_of_docs, gen1) - self.assertEqual(number_of_docs, gen2) - self.assertEqual(number_of_docs, len(doclist1)) - self.assertEqual(number_of_docs, len(doclist2)) - self.assertEqual(doclist1[0], doclist2[0]) - - d.addCallback(_assertDocSyncedFromDb1ToDb2) - - def _cleanUp(results): - db.delete_database() - db.close() - sol1.close() - sol2.close() - - d.addCallback(_cleanUp) - - return d - - def test_encrypted_sym_sync(self): - return self._test_encrypted_sym_sync() - - def test_encrypted_sym_sync_with_unicode_passphrase(self): - """ - Test the complete syncing chain between two soledad dbs using a - Soledad server backed by a couch database, using an unicode - passphrase. - """ - return self._test_encrypted_sym_sync(passphrase=u'ãáàäéàëíìïóòöõúùüñç') - - def test_sync_very_large_files(self): - """ - Test if Soledad can sync very large files. - """ - self.skipTest( - "Work in progress. For reference, see: " - "https://leap.se/code/issues/7370") - length = 100 * (10 ** 6) # 100 MB - return self._test_encrypted_sym_sync(doc_size=length, number_of_docs=1) - - def test_sync_many_small_files(self): - """ - Test if Soledad can sync many smallfiles. - """ - return self._test_encrypted_sym_sync(doc_size=2, number_of_docs=100) - - -class ConfigurationParsingTest(unittest.TestCase): - - def setUp(self): - self.maxDiff = None - - def test_use_defaults_on_failure(self): - config = load_configuration('this file will never exist') - expected = CONFIG_DEFAULTS - self.assertEquals(expected, config) - - def test_security_values_configuration(self): - # given - config_path = resource_filename('leap.soledad.common.tests', - 'fixture_soledad.conf') - # when - config = load_configuration(config_path) - - # then - expected = {'members': ['user1', 'user2'], - 'members_roles': ['role1', 'role2'], - 'admins': ['user3', 'user4'], - 'admins_roles': ['role3', 'role3']} - self.assertDictEqual(expected, config['database-security']) - - def test_server_values_configuration(self): - # given - config_path = resource_filename('leap.soledad.common.tests', - 'fixture_soledad.conf') - # when - config = load_configuration(config_path) - - # then - expected = {'couch_url': - 'http://soledad:passwd@localhost:5984', - 'create_cmd': - 'sudo -u soledad-admin /usr/bin/create-user-db', - 'admin_netrc': - '/etc/couchdb/couchdb-soledad-admin.netrc', - 'batching': False} - self.assertDictEqual(expected, config['soledad-server']) diff --git a/common/src/leap/soledad/common/tests/test_soledad.py b/common/src/leap/soledad/common/tests/test_soledad.py deleted file mode 100644 index aa52a733..00000000 --- a/common/src/leap/soledad/common/tests/test_soledad.py +++ /dev/null @@ -1,356 +0,0 @@ -# -*- coding: utf-8 -*- -# test_soledad.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Tests for general Soledad functionality. -""" -import os - -from mock import Mock - -from twisted.internet import defer - -from leap.common.events import catalog -from leap.soledad.common.tests.util import ( - BaseSoledadTest, - ADDRESS, -) -from leap import soledad -from leap.soledad.common.document import SoledadDocument -from leap.soledad.common.errors import DatabaseAccessError -from leap.soledad.client import Soledad -from leap.soledad.client.adbapi import U1DBConnectionPool -from leap.soledad.client.secrets import PassphraseTooShort -from leap.soledad.client.shared_db import SoledadSharedDatabase - - -class AuxMethodsTestCase(BaseSoledadTest): - - def test__init_dirs(self): - sol = self._soledad_instance(prefix='_init_dirs') - local_db_dir = os.path.dirname(sol.local_db_path) - secrets_path = os.path.dirname(sol.secrets.secrets_path) - self.assertTrue(os.path.isdir(local_db_dir)) - self.assertTrue(os.path.isdir(secrets_path)) - - def _close_soledad(results): - sol.close() - - d = sol.create_doc({}) - d.addCallback(_close_soledad) - return d - - def test__init_u1db_sqlcipher_backend(self): - sol = self._soledad_instance(prefix='_init_db') - self.assertIsInstance(sol._dbpool, U1DBConnectionPool) - self.assertTrue(os.path.isfile(sol.local_db_path)) - sol.close() - - def test__init_config_with_defaults(self): - """ - Test if configuration defaults point to the correct place. - """ - - class SoledadMock(Soledad): - - def __init__(self): - pass - - # instantiate without initializing so we just test - # _init_config_with_defaults() - sol = SoledadMock() - sol._passphrase = u'' - sol._server_url = '' - sol._init_config_with_defaults() - # assert value of local_db_path - self.assertEquals( - os.path.join(sol.default_prefix, 'soledad.u1db'), - sol.local_db_path) - - def test__init_config_from_params(self): - """ - Test if configuration is correctly read from file. - """ - sol = self._soledad_instance( - 'leap@leap.se', - passphrase=u'123', - secrets_path='value_3', - local_db_path='value_2', - server_url='value_1', - cert_file=None) - self.assertEqual( - os.path.join(self.tempdir, 'value_3'), - sol.secrets.secrets_path) - self.assertEqual( - os.path.join(self.tempdir, 'value_2'), - sol.local_db_path) - self.assertEqual('value_1', sol._server_url) - sol.close() - - @defer.inlineCallbacks - def test_change_passphrase(self): - """ - Test if passphrase can be changed. - """ - prefix = '_change_passphrase' - sol = self._soledad_instance( - 'leap@leap.se', - passphrase=u'123', - prefix=prefix, - ) - - doc1 = yield sol.create_doc({'simple': 'doc'}) - sol.change_passphrase(u'654321') - sol.close() - - with self.assertRaises(DatabaseAccessError): - self._soledad_instance( - 'leap@leap.se', - passphrase=u'123', - prefix=prefix) - - sol2 = self._soledad_instance( - 'leap@leap.se', - passphrase=u'654321', - prefix=prefix) - doc2 = yield sol2.get_doc(doc1.doc_id) - - self.assertEqual(doc1, doc2) - - sol2.close() - - def test_change_passphrase_with_short_passphrase_raises(self): - """ - Test if attempt to change passphrase passing a short passphrase - raises. - """ - sol = self._soledad_instance( - 'leap@leap.se', - passphrase=u'123') - # check that soledad complains about new passphrase length - self.assertRaises( - PassphraseTooShort, - sol.change_passphrase, u'54321') - sol.close() - - def test_get_passphrase(self): - """ - Assert passphrase getter works fine. - """ - sol = self._soledad_instance() - self.assertEqual('123', sol._passphrase) - sol.close() - - -class SoledadSharedDBTestCase(BaseSoledadTest): - - """ - These tests ensure the functionalities of the shared recovery database. - """ - - def setUp(self): - BaseSoledadTest.setUp(self) - self._shared_db = SoledadSharedDatabase( - 'https://provider/', ADDRESS, document_factory=SoledadDocument, - creds=None) - - def tearDown(self): - BaseSoledadTest.tearDown(self) - - def test__get_secrets_from_shared_db(self): - """ - Ensure the shared db is queried with the correct doc_id. - """ - doc_id = self._soledad.secrets._shared_db_doc_id() - self._soledad.secrets._get_secrets_from_shared_db() - self.assertTrue( - self._soledad.shared_db.get_doc.assert_called_with( - doc_id) is None, - 'Wrong doc_id when fetching recovery document.') - - def test__put_secrets_in_shared_db(self): - """ - Ensure recovery document is put into shared recover db. - """ - doc_id = self._soledad.secrets._shared_db_doc_id() - self._soledad.secrets._put_secrets_in_shared_db() - self.assertTrue( - self._soledad.shared_db.get_doc.assert_called_with( - doc_id) is None, - 'Wrong doc_id when fetching recovery document.') - self.assertTrue( - self._soledad.shared_db.put_doc.assert_called_with( - self._doc_put) is None, - 'Wrong document when putting recovery document.') - self.assertTrue( - self._doc_put.doc_id == doc_id, - 'Wrong doc_id when putting recovery document.') - - -class SoledadSignalingTestCase(BaseSoledadTest): - - """ - These tests ensure signals are correctly emmited by Soledad. - """ - - EVENTS_SERVER_PORT = 8090 - - def setUp(self): - # mock signaling - soledad.client.signal = Mock() - soledad.client.secrets.events.emit_async = Mock() - # run parent's setUp - BaseSoledadTest.setUp(self) - - def tearDown(self): - BaseSoledadTest.tearDown(self) - - def _pop_mock_call(self, mocked): - mocked.call_args_list.pop() - mocked.mock_calls.pop() - mocked.call_args = mocked.call_args_list[-1] - - def test_stage3_bootstrap_signals(self): - """ - Test that a fresh soledad emits all bootstrap signals. - - Signals are: - - downloading keys / done downloading keys. - - creating keys / done creating keys. - - downloading keys / done downloading keys. - - uploading keys / done uploading keys. - """ - soledad.client.secrets.events.emit_async.reset_mock() - # get a fresh instance so it emits all bootstrap signals - sol = self._soledad_instance( - secrets_path='alternative_stage3.json', - local_db_path='alternative_stage3.u1db') - # reverse call order so we can verify in the order the signals were - # expected - soledad.client.secrets.events.emit_async.mock_calls.reverse() - soledad.client.secrets.events.emit_async.call_args = \ - soledad.client.secrets.events.emit_async.call_args_list[0] - soledad.client.secrets.events.emit_async.call_args_list.reverse() - - user_data = {'userid': ADDRESS, 'uuid': ADDRESS} - - # downloading keys signals - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DOWNLOADING_KEYS, user_data - ) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data - ) - # creating keys signals - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_CREATING_KEYS, user_data - ) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DONE_CREATING_KEYS, user_data - ) - # downloading once more (inside _put_keys_in_shared_db) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DOWNLOADING_KEYS, user_data - ) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data - ) - # uploading keys signals - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_UPLOADING_KEYS, user_data - ) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DONE_UPLOADING_KEYS, user_data - ) - sol.close() - - def test_stage2_bootstrap_signals(self): - """ - Test that if there are keys in server, soledad will download them and - emit corresponding signals. - """ - # get existing instance so we have access to keys - sol = self._soledad_instance() - # create a document with secrets - doc = SoledadDocument(doc_id=sol.secrets._shared_db_doc_id()) - doc.content = sol.secrets._export_recovery_document() - sol.close() - # reset mock - soledad.client.secrets.events.emit_async.reset_mock() - # get a fresh instance so it emits all bootstrap signals - shared_db = self.get_default_shared_mock(get_doc_return_value=doc) - sol = self._soledad_instance( - secrets_path='alternative_stage2.json', - local_db_path='alternative_stage2.u1db', - shared_db_class=shared_db) - # reverse call order so we can verify in the order the signals were - # expected - soledad.client.secrets.events.emit_async.mock_calls.reverse() - soledad.client.secrets.events.emit_async.call_args = \ - soledad.client.secrets.events.emit_async.call_args_list[0] - soledad.client.secrets.events.emit_async.call_args_list.reverse() - # assert download keys signals - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DOWNLOADING_KEYS, - {'userid': ADDRESS, 'uuid': ADDRESS} - ) - self._pop_mock_call(soledad.client.secrets.events.emit_async) - soledad.client.secrets.events.emit_async.assert_called_with( - catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, - {'userid': ADDRESS, 'uuid': ADDRESS}, - ) - sol.close() - - def test_stage1_bootstrap_signals(self): - """ - Test that if soledad already has a local secret, it emits no signals. - """ - soledad.client.signal.reset_mock() - # get an existent instance so it emits only some of bootstrap signals - sol = self._soledad_instance() - self.assertEqual([], soledad.client.signal.mock_calls) - sol.close() - - @defer.inlineCallbacks - def test_sync_signals(self): - """ - Test Soledad emits SOLEDAD_CREATING_KEYS signal. - """ - # get a fresh instance so it emits all bootstrap signals - sol = self._soledad_instance() - soledad.client.signal.reset_mock() - - # mock the actual db sync so soledad does not try to connect to the - # server - d = defer.Deferred() - d.callback(None) - sol._dbsyncer.sync = Mock(return_value=d) - - yield sol.sync() - - # assert the signal has been emitted - soledad.client.events.emit_async.assert_called_with( - catalog.SOLEDAD_DONE_DATA_SYNC, - {'userid': ADDRESS, 'uuid': ADDRESS}, - ) - sol.close() diff --git a/common/src/leap/soledad/common/tests/test_soledad_app.py b/common/src/leap/soledad/common/tests/test_soledad_app.py deleted file mode 100644 index 7f9a58d3..00000000 --- a/common/src/leap/soledad/common/tests/test_soledad_app.py +++ /dev/null @@ -1,54 +0,0 @@ -# -*- coding: utf-8 -*- -# test_soledad_app.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Test ObjectStore and Couch backend bits. -""" -from testscenarios import TestWithScenarios - -from leap.soledad.common.tests.util import BaseSoledadTest -from leap.soledad.common.tests.util import make_soledad_document_for_test -from leap.soledad.common.tests.util import make_soledad_app -from leap.soledad.common.tests.util import make_token_soledad_app -from leap.soledad.common.tests.util import make_token_http_database_for_test -from leap.soledad.common.tests.util import copy_token_http_database_for_test -from leap.soledad.common.tests.u1db_tests import test_backends - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_backends`. -# ----------------------------------------------------------------------------- - -LEAP_SCENARIOS = [ - ('http', { - 'make_database_for_test': test_backends.make_http_database_for_test, - 'copy_database_for_test': test_backends.copy_http_database_for_test, - 'make_document_for_test': make_soledad_document_for_test, - 'make_app_with_state': make_soledad_app}), -] - - -class SoledadTests( - TestWithScenarios, test_backends.AllDatabaseTests, BaseSoledadTest): - - scenarios = LEAP_SCENARIOS + [ - ('token_http', { - 'make_database_for_test': make_token_http_database_for_test, - 'copy_database_for_test': copy_token_http_database_for_test, - 'make_document_for_test': make_soledad_document_for_test, - 'make_app_with_state': make_token_soledad_app, - }) - ] diff --git a/common/src/leap/soledad/common/tests/test_soledad_doc.py b/common/src/leap/soledad/common/tests/test_soledad_doc.py deleted file mode 100644 index df9fd09e..00000000 --- a/common/src/leap/soledad/common/tests/test_soledad_doc.py +++ /dev/null @@ -1,46 +0,0 @@ -# -*- coding: utf-8 -*- -# test_soledad_doc.py -# Copyright (C) 2013, 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Test Leap backend bits: soledad docs -""" -from testscenarios import TestWithScenarios - -from leap.soledad.common.tests.u1db_tests import test_document -from leap.soledad.common.tests.util import BaseSoledadTest -from leap.soledad.common.tests.util import make_soledad_document_for_test - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_document`. -# ----------------------------------------------------------------------------- - -class TestSoledadDocument( - TestWithScenarios, - test_document.TestDocument, BaseSoledadTest): - - scenarios = ([( - 'leap', { - 'make_document_for_test': make_soledad_document_for_test})]) - - -class TestSoledadPyDocument( - TestWithScenarios, - test_document.TestPyDocument, BaseSoledadTest): - - scenarios = ([( - 'leap', { - 'make_document_for_test': make_soledad_document_for_test})]) diff --git a/common/src/leap/soledad/common/tests/test_sqlcipher.py b/common/src/leap/soledad/common/tests/test_sqlcipher.py deleted file mode 100644 index 2bcdf0fb..00000000 --- a/common/src/leap/soledad/common/tests/test_sqlcipher.py +++ /dev/null @@ -1,721 +0,0 @@ -# -*- coding: utf-8 -*- -# test_sqlcipher.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Test sqlcipher backend internals. -""" -import os -import time -import threading -import tempfile -import shutil - -from pysqlcipher import dbapi2 -from testscenarios import TestWithScenarios - -# l2db stuff. -from leap.soledad.common.l2db import errors -from leap.soledad.common.l2db import query_parser -from leap.soledad.common.l2db.backends.sqlite_backend import SQLitePartialExpandDatabase - -# soledad stuff. -from leap.soledad.common import soledad_assert -from leap.soledad.common.document import SoledadDocument -from leap.soledad.client.sqlcipher import SQLCipherDatabase -from leap.soledad.client.sqlcipher import SQLCipherOptions -from leap.soledad.client.sqlcipher import DatabaseIsNotEncrypted - -# u1db tests stuff. -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests.u1db_tests import test_backends -from leap.soledad.common.tests.u1db_tests import test_open -from leap.soledad.common.tests.util import make_sqlcipher_database_for_test -from leap.soledad.common.tests.util import copy_sqlcipher_database_for_test -from leap.soledad.common.tests.util import PASSWORD -from leap.soledad.common.tests.util import BaseSoledadTest - - -def sqlcipher_open(path, passphrase, create=True, document_factory=None): - return SQLCipherDatabase( - SQLCipherOptions(path, passphrase, create=create)) - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_common_backend`. -# ----------------------------------------------------------------------------- - -class TestSQLCipherBackendImpl(tests.TestCase): - - def test__allocate_doc_id(self): - db = sqlcipher_open(':memory:', PASSWORD) - doc_id1 = db._allocate_doc_id() - self.assertTrue(doc_id1.startswith('D-')) - self.assertEqual(34, len(doc_id1)) - int(doc_id1[len('D-'):], 16) - self.assertNotEqual(doc_id1, db._allocate_doc_id()) - db.close() - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_backends`. -# ----------------------------------------------------------------------------- - -def make_document_for_test(test, doc_id, rev, content, has_conflicts=False): - return SoledadDocument(doc_id, rev, content, has_conflicts=has_conflicts) - - -SQLCIPHER_SCENARIOS = [ - ('sqlcipher', {'make_database_for_test': make_sqlcipher_database_for_test, - 'copy_database_for_test': copy_sqlcipher_database_for_test, - 'make_document_for_test': make_document_for_test, }), -] - - -class SQLCipherTests(TestWithScenarios, test_backends.AllDatabaseTests): - scenarios = SQLCIPHER_SCENARIOS - - -class SQLCipherDatabaseTests(TestWithScenarios, - test_backends.LocalDatabaseTests): - scenarios = SQLCIPHER_SCENARIOS - - -class SQLCipherValidateGenNTransIdTests( - TestWithScenarios, - test_backends.LocalDatabaseValidateGenNTransIdTests): - scenarios = SQLCIPHER_SCENARIOS - - -class SQLCipherValidateSourceGenTests( - TestWithScenarios, - test_backends.LocalDatabaseValidateSourceGenTests): - scenarios = SQLCIPHER_SCENARIOS - - -class SQLCipherWithConflictsTests( - TestWithScenarios, - test_backends.LocalDatabaseWithConflictsTests): - scenarios = SQLCIPHER_SCENARIOS - - -class SQLCipherIndexTests( - TestWithScenarios, test_backends.DatabaseIndexTests): - scenarios = SQLCIPHER_SCENARIOS - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_sqlite_backend`. -# ----------------------------------------------------------------------------- - -class TestSQLCipherDatabase(tests.TestCase): - """ - Tests from u1db.tests.test_sqlite_backend.TestSQLiteDatabase. - """ - - def test_atomic_initialize(self): - # This test was modified to ensure that db2.close() is called within - # the thread that created the database. - tmpdir = self.createTempDir() - dbname = os.path.join(tmpdir, 'atomic.db') - - t2 = None # will be a thread - - class SQLCipherDatabaseTesting(SQLCipherDatabase): - _index_storage_value = "testing" - - def __init__(self, dbname, ntry): - self._try = ntry - self._is_initialized_invocations = 0 - SQLCipherDatabase.__init__( - self, - SQLCipherOptions(dbname, PASSWORD)) - - def _is_initialized(self, c): - res = \ - SQLCipherDatabase._is_initialized(self, c) - if self._try == 1: - self._is_initialized_invocations += 1 - if self._is_initialized_invocations == 2: - t2.start() - # hard to do better and have a generic test - time.sleep(0.05) - return res - - class SecondTry(threading.Thread): - - outcome2 = [] - - def run(self): - try: - db2 = SQLCipherDatabaseTesting(dbname, 2) - except Exception, e: - SecondTry.outcome2.append(e) - else: - SecondTry.outcome2.append(db2) - - t2 = SecondTry() - db1 = SQLCipherDatabaseTesting(dbname, 1) - t2.join() - - self.assertIsInstance(SecondTry.outcome2[0], SQLCipherDatabaseTesting) - self.assertTrue(db1._is_initialized(db1._get_sqlite_handle().cursor())) - db1.close() - - -class TestAlternativeDocument(SoledadDocument): - - """A (not very) alternative implementation of Document.""" - - -class TestSQLCipherPartialExpandDatabase(tests.TestCase): - """ - Tests from u1db.tests.test_sqlite_backend.TestSQLitePartialExpandDatabase. - """ - - # The following tests had to be cloned from u1db because they all - # instantiate the backend directly, so we need to change that in order to - # our backend be instantiated in place. - - def setUp(self): - self.db = sqlcipher_open(':memory:', PASSWORD) - - def tearDown(self): - self.db.close() - - def test_default_replica_uid(self): - self.assertIsNot(None, self.db._replica_uid) - self.assertEqual(32, len(self.db._replica_uid)) - int(self.db._replica_uid, 16) - - def test__parse_index(self): - g = self.db._parse_index_definition('fieldname') - self.assertIsInstance(g, query_parser.ExtractField) - self.assertEqual(['fieldname'], g.field) - - def test__update_indexes(self): - g = self.db._parse_index_definition('fieldname') - c = self.db._get_sqlite_handle().cursor() - self.db._update_indexes('doc-id', {'fieldname': 'val'}, - [('fieldname', g)], c) - c.execute('SELECT doc_id, field_name, value FROM document_fields') - self.assertEqual([('doc-id', 'fieldname', 'val')], - c.fetchall()) - - def test_create_database(self): - raw_db = self.db._get_sqlite_handle() - self.assertNotEqual(None, raw_db) - - def test__set_replica_uid(self): - # Start from scratch, so that replica_uid isn't set. - self.assertIsNot(None, self.db._real_replica_uid) - self.assertIsNot(None, self.db._replica_uid) - self.db._set_replica_uid('foo') - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT value FROM u1db_config WHERE name='replica_uid'") - self.assertEqual(('foo',), c.fetchone()) - self.assertEqual('foo', self.db._real_replica_uid) - self.assertEqual('foo', self.db._replica_uid) - self.db._close_sqlite_handle() - self.assertEqual('foo', self.db._replica_uid) - - def test__open_database(self): - # SQLCipherDatabase has no _open_database() method, so we just pass - # (and test for the same funcionality on test_open_database_existing() - # below). - pass - - def test__open_database_with_factory(self): - # SQLCipherDatabase has no _open_database() method. - pass - - def test__open_database_non_existent(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/non-existent.sqlite' - self.assertRaises(errors.DatabaseDoesNotExist, - sqlcipher_open, - path, PASSWORD, create=False) - - def test__open_database_during_init(self): - # The purpose of this test is to ensure that _open_database() parallel - # db initialization behaviour is correct. As SQLCipherDatabase does - # not have an _open_database() method, we just do not implement this - # test. - pass - - def test__open_database_invalid(self): - # This test was modified to ensure that an empty database file will - # raise a DatabaseIsNotEncrypted exception instead of a - # dbapi2.OperationalError exception. - temp_dir = self.createTempDir(prefix='u1db-test-') - path1 = temp_dir + '/invalid1.db' - with open(path1, 'wb') as f: - f.write("") - self.assertRaises(DatabaseIsNotEncrypted, - sqlcipher_open, path1, - PASSWORD) - with open(path1, 'wb') as f: - f.write("invalid") - self.assertRaises(dbapi2.DatabaseError, - sqlcipher_open, path1, - PASSWORD) - - def test_open_database_existing(self): - # In the context of SQLCipherDatabase, where no _open_database() - # method exists and thus there's no call to _which_index_storage(), - # this test tests for the same functionality as - # test_open_database_create() below. So, we just pass. - pass - - def test_open_database_with_factory(self): - # SQLCipherDatabase's constructor has no factory parameter. - pass - - def test_open_database_create(self): - # SQLCipherDatabas has no open_database() method, so we just test for - # the actual database constructor effects. - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/new.sqlite' - db1 = sqlcipher_open(path, PASSWORD, create=True) - db2 = sqlcipher_open(path, PASSWORD, create=False) - self.assertIsInstance(db2, SQLCipherDatabase) - db1.close() - db2.close() - - def test_create_database_initializes_schema(self): - # This test had to be cloned because our implementation of SQLCipher - # backend is referenced with an index_storage_value that includes the - # word "encrypted". See u1db's sqlite_backend and our - # sqlcipher_backend for reference. - raw_db = self.db._get_sqlite_handle() - c = raw_db.cursor() - c.execute("SELECT * FROM u1db_config") - config = dict([(r[0], r[1]) for r in c.fetchall()]) - replica_uid = self.db._replica_uid - self.assertEqual({'sql_schema': '0', 'replica_uid': replica_uid, - 'index_storage': 'expand referenced encrypted'}, - config) - - def test_store_syncable(self): - doc = self.db.create_doc_from_json(tests.simple_doc) - # assert that docs are syncable by default - self.assertEqual(True, doc.syncable) - # assert that we can store syncable = False - doc.syncable = False - self.db.put_doc(doc) - self.assertEqual(False, self.db.get_doc(doc.doc_id).syncable) - # assert that we can store syncable = True - doc.syncable = True - self.db.put_doc(doc) - self.assertEqual(True, self.db.get_doc(doc.doc_id).syncable) - - def test__close_sqlite_handle(self): - raw_db = self.db._get_sqlite_handle() - self.db._close_sqlite_handle() - self.assertRaises(dbapi2.ProgrammingError, - raw_db.cursor) - - def test__get_generation(self): - self.assertEqual(0, self.db._get_generation()) - - def test__get_generation_info(self): - self.assertEqual((0, ''), self.db._get_generation_info()) - - def test_create_index(self): - self.db.create_index('test-idx', "key") - self.assertEqual([('test-idx', ["key"])], self.db.list_indexes()) - - def test_create_index_multiple_fields(self): - self.db.create_index('test-idx', "key", "key2") - self.assertEqual([('test-idx', ["key", "key2"])], - self.db.list_indexes()) - - def test__get_index_definition(self): - self.db.create_index('test-idx', "key", "key2") - # TODO: How would you test that an index is getting used for an SQL - # request? - self.assertEqual(["key", "key2"], - self.db._get_index_definition('test-idx')) - - def test_list_index_mixed(self): - # Make sure that we properly order the output - c = self.db._get_sqlite_handle().cursor() - # We intentionally insert the data in weird ordering, to make sure the - # query still gets it back correctly. - c.executemany("INSERT INTO index_definitions VALUES (?, ?, ?)", - [('idx-1', 0, 'key10'), - ('idx-2', 2, 'key22'), - ('idx-1', 1, 'key11'), - ('idx-2', 0, 'key20'), - ('idx-2', 1, 'key21')]) - self.assertEqual([('idx-1', ['key10', 'key11']), - ('idx-2', ['key20', 'key21', 'key22'])], - self.db.list_indexes()) - - def test_no_indexes_no_document_fields(self): - self.db.create_doc_from_json( - '{"key1": "val1", "key2": "val2"}') - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([], c.fetchall()) - - def test_create_extracts_fields(self): - doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') - doc2 = self.db.create_doc_from_json('{"key1": "valx", "key2": "valy"}') - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([], c.fetchall()) - self.db.create_index('test', 'key1', 'key2') - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual(sorted( - [(doc1.doc_id, "key1", "val1"), - (doc1.doc_id, "key2", "val2"), - (doc2.doc_id, "key1", "valx"), - (doc2.doc_id, "key2", "valy"), ]), sorted(c.fetchall())) - - def test_put_updates_fields(self): - self.db.create_index('test', 'key1', 'key2') - doc1 = self.db.create_doc_from_json( - '{"key1": "val1", "key2": "val2"}') - doc1.content = {"key1": "val1", "key2": "valy"} - self.db.put_doc(doc1) - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([(doc1.doc_id, "key1", "val1"), - (doc1.doc_id, "key2", "valy"), ], c.fetchall()) - - def test_put_updates_nested_fields(self): - self.db.create_index('test', 'key', 'sub.doc') - doc1 = self.db.create_doc_from_json(tests.nested_doc) - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([(doc1.doc_id, "key", "value"), - (doc1.doc_id, "sub.doc", "underneath"), ], - c.fetchall()) - - def test__ensure_schema_rollback(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/rollback.db' - - class SQLitePartialExpandDbTesting(SQLCipherDatabase): - - def _set_replica_uid_in_transaction(self, uid): - super(SQLitePartialExpandDbTesting, - self)._set_replica_uid_in_transaction(uid) - if fail: - raise Exception() - - db = SQLitePartialExpandDbTesting.__new__(SQLitePartialExpandDbTesting) - db._db_handle = dbapi2.connect(path) # db is there but not yet init-ed - fail = True - self.assertRaises(Exception, db._ensure_schema) - fail = False - db._initialize(db._db_handle.cursor()) - - def test_open_database_non_existent(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/non-existent.sqlite' - self.assertRaises(errors.DatabaseDoesNotExist, - sqlcipher_open, path, "123", - create=False) - - def test_delete_database_existent(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/new.sqlite' - db = sqlcipher_open(path, "123", create=True) - db.close() - SQLCipherDatabase.delete_database(path) - self.assertRaises(errors.DatabaseDoesNotExist, - sqlcipher_open, path, "123", - create=False) - - def test_delete_database_nonexistent(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/non-existent.sqlite' - self.assertRaises(errors.DatabaseDoesNotExist, - SQLCipherDatabase.delete_database, path) - - def test__get_indexed_fields(self): - self.db.create_index('idx1', 'a', 'b') - self.assertEqual(set(['a', 'b']), self.db._get_indexed_fields()) - self.db.create_index('idx2', 'b', 'c') - self.assertEqual(set(['a', 'b', 'c']), self.db._get_indexed_fields()) - - def test_indexed_fields_expanded(self): - self.db.create_index('idx1', 'key1') - doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') - self.assertEqual(set(['key1']), self.db._get_indexed_fields()) - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([(doc1.doc_id, 'key1', 'val1')], c.fetchall()) - - def test_create_index_updates_fields(self): - doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') - self.db.create_index('idx1', 'key1') - self.assertEqual(set(['key1']), self.db._get_indexed_fields()) - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([(doc1.doc_id, 'key1', 'val1')], c.fetchall()) - - def assertFormatQueryEquals(self, exp_statement, exp_args, definition, - values): - statement, args = self.db._format_query(definition, values) - self.assertEqual(exp_statement, statement) - self.assertEqual(exp_args, args) - - def test__format_query(self): - self.assertFormatQueryEquals( - "SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM " - "document d, document_fields d0 LEFT OUTER JOIN conflicts c ON " - "c.doc_id = d.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name " - "= ? AND d0.value = ? GROUP BY d.doc_id, d.doc_rev, d.content " - "ORDER BY d0.value;", ["key1", "a"], - ["key1"], ["a"]) - - def test__format_query2(self): - self.assertFormatQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value = ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value = ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value = ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' - 'd0.value, d1.value, d2.value;', - ["key1", "a", "key2", "b", "key3", "c"], - ["key1", "key2", "key3"], ["a", "b", "c"]) - - def test__format_query_wildcard(self): - self.assertFormatQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value = ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value GLOB ? AND d.doc_id = d2.doc_id AND d2.field_name = ? ' - 'AND d2.value NOT NULL GROUP BY d.doc_id, d.doc_rev, d.content ' - 'ORDER BY d0.value, d1.value, d2.value;', - ["key1", "a", "key2", "b*", "key3"], ["key1", "key2", "key3"], - ["a", "b*", "*"]) - - def assertFormatRangeQueryEquals(self, exp_statement, exp_args, definition, - start_value, end_value): - statement, args = self.db._format_range_query( - definition, start_value, end_value) - self.assertEqual(exp_statement, statement) - self.assertEqual(exp_args, args) - - def test__format_range_query(self): - self.assertFormatRangeQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value >= ? AND d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value <= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value <= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' - 'd0.value, d1.value, d2.value;', - ['key1', 'a', 'key2', 'b', 'key3', 'c', 'key1', 'p', 'key2', 'q', - 'key3', 'r'], - ["key1", "key2", "key3"], ["a", "b", "c"], ["p", "q", "r"]) - - def test__format_range_query_no_start(self): - self.assertFormatRangeQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value <= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value <= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' - 'd0.value, d1.value, d2.value;', - ['key1', 'a', 'key2', 'b', 'key3', 'c'], - ["key1", "key2", "key3"], None, ["a", "b", "c"]) - - def test__format_range_query_no_end(self): - self.assertFormatRangeQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value >= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' - 'd0.value, d1.value, d2.value;', - ['key1', 'a', 'key2', 'b', 'key3', 'c'], - ["key1", "key2", "key3"], ["a", "b", "c"], None) - - def test__format_range_query_wildcard(self): - self.assertFormatRangeQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value NOT NULL AND d.doc_id = d0.doc_id AND d0.field_name = ? ' - 'AND d0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? ' - 'AND (d1.value < ? OR d1.value GLOB ?) AND d.doc_id = d2.doc_id ' - 'AND d2.field_name = ? AND d2.value NOT NULL GROUP BY d.doc_id, ' - 'd.doc_rev, d.content ORDER BY d0.value, d1.value, d2.value;', - ['key1', 'a', 'key2', 'b', 'key3', 'key1', 'p', 'key2', 'q', 'q*', - 'key3'], - ["key1", "key2", "key3"], ["a", "b*", "*"], ["p", "q*", "*"]) - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_open`. -# ----------------------------------------------------------------------------- - - -class SQLCipherOpen(test_open.TestU1DBOpen): - - def test_open_no_create(self): - self.assertRaises(errors.DatabaseDoesNotExist, - sqlcipher_open, self.db_path, - PASSWORD, - create=False) - self.assertFalse(os.path.exists(self.db_path)) - - def test_open_create(self): - db = sqlcipher_open(self.db_path, PASSWORD, create=True) - self.addCleanup(db.close) - self.assertTrue(os.path.exists(self.db_path)) - self.assertIsInstance(db, SQLCipherDatabase) - - def test_open_with_factory(self): - db = sqlcipher_open(self.db_path, PASSWORD, create=True, - document_factory=TestAlternativeDocument) - self.addCleanup(db.close) - doc = db.create_doc({}) - self.assertTrue(isinstance(doc, SoledadDocument)) - - def test_open_existing(self): - db = sqlcipher_open(self.db_path, PASSWORD) - self.addCleanup(db.close) - doc = db.create_doc_from_json(tests.simple_doc) - # Even though create=True, we shouldn't wipe the db - db2 = sqlcipher_open(self.db_path, PASSWORD, create=True) - self.addCleanup(db2.close) - doc2 = db2.get_doc(doc.doc_id) - self.assertEqual(doc, doc2) - - def test_open_existing_no_create(self): - db = sqlcipher_open(self.db_path, PASSWORD) - self.addCleanup(db.close) - db2 = sqlcipher_open(self.db_path, PASSWORD, create=False) - self.addCleanup(db2.close) - self.assertIsInstance(db2, SQLCipherDatabase) - - -# ----------------------------------------------------------------------------- -# Tests for actual encryption of the database -# ----------------------------------------------------------------------------- - -class SQLCipherEncryptionTests(BaseSoledadTest): - - """ - Tests to guarantee SQLCipher is indeed encrypting data when storing. - """ - - def _delete_dbfiles(self): - for dbfile in [self.DB_FILE]: - if os.path.exists(dbfile): - os.unlink(dbfile) - - def setUp(self): - # the following come from BaseLeapTest.setUpClass, because - # twisted.trial doesn't support such class methods for setting up - # test classes. - self.old_path = os.environ['PATH'] - self.old_home = os.environ['HOME'] - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.home = self.tempdir - bin_tdir = os.path.join( - self.tempdir, - 'bin') - os.environ["PATH"] = bin_tdir - os.environ["HOME"] = self.tempdir - # this is our own stuff - self.DB_FILE = os.path.join(self.tempdir, 'test.db') - self._delete_dbfiles() - - def tearDown(self): - self._delete_dbfiles() - # the following come from BaseLeapTest.tearDownClass, because - # twisted.trial doesn't support such class methods for tearing down - # test classes. - os.environ["PATH"] = self.old_path - os.environ["HOME"] = self.old_home - # safety check! please do not wipe my home... - # XXX needs to adapt to non-linuces - soledad_assert( - self.tempdir.startswith('/tmp/leap_tests-') or - self.tempdir.startswith('/var/folder'), - "beware! tried to remove a dir which does not " - "live in temporal folder!") - shutil.rmtree(self.tempdir) - - def test_try_to_open_encrypted_db_with_sqlite_backend(self): - """ - SQLite backend should not succeed to open SQLCipher databases. - """ - db = sqlcipher_open(self.DB_FILE, PASSWORD) - doc = db.create_doc_from_json(tests.simple_doc) - db.close() - try: - # trying to open an encrypted database with the regular u1db - # backend should raise a DatabaseError exception. - SQLitePartialExpandDatabase(self.DB_FILE, - document_factory=SoledadDocument) - raise DatabaseIsNotEncrypted() - except dbapi2.DatabaseError: - # at this point we know that the regular U1DB sqlcipher backend - # did not succeed on opening the database, so it was indeed - # encrypted. - db = sqlcipher_open(self.DB_FILE, PASSWORD) - doc = db.get_doc(doc.doc_id) - self.assertEqual(tests.simple_doc, doc.get_json(), - 'decrypted content mismatch') - db.close() - - def test_try_to_open_raw_db_with_sqlcipher_backend(self): - """ - SQLCipher backend should not succeed to open unencrypted databases. - """ - db = SQLitePartialExpandDatabase(self.DB_FILE, - document_factory=SoledadDocument) - db.create_doc_from_json(tests.simple_doc) - db.close() - try: - # trying to open the a non-encrypted database with sqlcipher - # backend should raise a DatabaseIsNotEncrypted exception. - db = sqlcipher_open(self.DB_FILE, PASSWORD) - db.close() - raise dbapi2.DatabaseError( - "SQLCipher backend should not be able to open non-encrypted " - "dbs.") - except DatabaseIsNotEncrypted: - pass diff --git a/common/src/leap/soledad/common/tests/test_sqlcipher_sync.py b/common/src/leap/soledad/common/tests/test_sqlcipher_sync.py deleted file mode 100644 index 42cfa6b7..00000000 --- a/common/src/leap/soledad/common/tests/test_sqlcipher_sync.py +++ /dev/null @@ -1,743 +0,0 @@ -# -*- coding: utf-8 -*- -# test_sqlcipher.py -# Copyright (C) 2013-2016 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Test sqlcipher backend sync. -""" -import os - -from uuid import uuid4 - -from testscenarios import TestWithScenarios - -from leap.soledad.common.l2db import sync -from leap.soledad.common.l2db import vectorclock -from leap.soledad.common.l2db import errors - -from leap.soledad.common.crypto import ENC_SCHEME_KEY -from leap.soledad.client.crypto import decrypt_doc_dict -from leap.soledad.client.http_target import SoledadHTTPSyncTarget - -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests.test_sqlcipher import SQLCIPHER_SCENARIOS -from leap.soledad.common.tests.util import make_soledad_app -from leap.soledad.common.tests.test_sync_target import \ - SoledadDatabaseSyncTargetTests -from leap.soledad.common.tests.util import soledad_sync_target -from leap.soledad.common.tests.util import BaseSoledadTest - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_sync`. -# ----------------------------------------------------------------------------- - -def sync_via_synchronizer_and_soledad(test, db_source, db_target, - trace_hook=None, - trace_hook_shallow=None): - if trace_hook: - test.skipTest("full trace hook unsupported over http") - path = test._http_at[db_target] - target = SoledadHTTPSyncTarget.connect( - test.getURL(path), test._soledad._crypto) - target.set_token_credentials('user-uuid', 'auth-token') - if trace_hook_shallow: - target._set_trace_hook_shallow(trace_hook_shallow) - return sync.Synchronizer(db_source, target).sync() - - -def sync_via_synchronizer(test, db_source, db_target, - trace_hook=None, - trace_hook_shallow=None): - target = db_target.get_sync_target() - trace_hook = trace_hook or trace_hook_shallow - if trace_hook: - target._set_trace_hook(trace_hook) - return sync.Synchronizer(db_source, target).sync() - - -sync_scenarios = [] -for name, scenario in SQLCIPHER_SCENARIOS: - scenario['do_sync'] = sync_via_synchronizer - sync_scenarios.append((name, scenario)) - - -class SQLCipherDatabaseSyncTests( - TestWithScenarios, - tests.DatabaseBaseTests, - BaseSoledadTest): - - """ - Test for succesfull sync between SQLCipher and LeapBackend. - - Some of the tests in this class had to be adapted because the remote - backend always receive encrypted content, and so it can not rely on - document's content comparison to try to autoresolve conflicts. - """ - - scenarios = sync_scenarios - - def setUp(self): - self._use_tracking = {} - super(tests.DatabaseBaseTests, self).setUp() - - def create_database(self, replica_uid, sync_role=None): - if replica_uid == 'test' and sync_role is None: - # created up the chain by base class but unused - return None - db = self.create_database_for_role(replica_uid, sync_role) - if sync_role: - self._use_tracking[db] = (replica_uid, sync_role) - self.addCleanup(db.close) - return db - - def create_database_for_role(self, replica_uid, sync_role): - # hook point for reuse - return tests.DatabaseBaseTests.create_database(self, replica_uid) - - def sync(self, db_from, db_to, trace_hook=None, - trace_hook_shallow=None): - from_name, from_sync_role = self._use_tracking[db_from] - to_name, to_sync_role = self._use_tracking[db_to] - if from_sync_role not in ('source', 'both'): - raise Exception("%s marked for %s use but used as source" % - (from_name, from_sync_role)) - if to_sync_role not in ('target', 'both'): - raise Exception("%s marked for %s use but used as target" % - (to_name, to_sync_role)) - return self.do_sync(self, db_from, db_to, trace_hook, - trace_hook_shallow) - - def assertLastExchangeLog(self, db, expected): - log = getattr(db, '_last_exchange_log', None) - if log is None: - return - self.assertEqual(expected, log) - - def copy_database(self, db, sync_role=None): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES - # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST - # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS - # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND - # NINJA TO YOUR HOUSE. - db_copy = tests.DatabaseBaseTests.copy_database(self, db) - name, orig_sync_role = self._use_tracking[db] - self._use_tracking[db_copy] = (name + '(copy)', sync_role or - orig_sync_role) - return db_copy - - def test_sync_tracks_db_generation_of_other(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.assertEqual(0, self.sync(self.db1, self.db2)) - self.assertEqual( - (0, ''), self.db1._get_replica_gen_and_trans_id('test2')) - self.assertEqual( - (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) - self.assertLastExchangeLog(self.db2, - {'receive': - {'docs': [], 'last_known_gen': 0}, - 'return': - {'docs': [], 'last_gen': 0}}) - - def test_sync_autoresolves(self): - """ - Test for sync autoresolve remote. - - This test was adapted because the remote database receives encrypted - content and so it can't compare documents contents to autoresolve. - """ - # The remote database can't autoresolve conflicts based on magic - # content convergence, so we modify this test to leave the possibility - # of the remode document ending up in conflicted state. - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc1 = self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc') - rev1 = doc1.rev - doc2 = self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc') - rev2 = doc2.rev - self.sync(self.db1, self.db2) - doc = self.db1.get_doc('doc') - self.assertFalse(doc.has_conflicts) - # if remote content is in conflicted state, then document revisions - # will be different. - # self.assertEqual(doc.rev, self.db2.get_doc('doc').rev) - v = vectorclock.VectorClockRev(doc.rev) - self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev1))) - self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev2))) - - def test_sync_autoresolves_moar(self): - """ - Test for sync autoresolve local. - - This test was adapted to decrypt remote content before assert. - """ - # here we test that when a database that has a conflicted document is - # the source of a sync, and the target database has a revision of the - # conflicted document that is newer than the source database's, and - # that target's database's document's content is the same as the - # source's document's conflict's, the source's document's conflict gets - # autoresolved, and the source's document's revision bumped. - # - # idea is as follows: - # A B - # a1 - - # `-------> - # a1 a1 - # v v - # a2 a1b1 - # `-------> - # a1b1+a2 a1b1 - # v - # a1b1+a2 a1b2 (a1b2 has same content as a2) - # `-------> - # a3b2 a1b2 (autoresolved) - # `-------> - # a3b2 a3b2 - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc') - self.sync(self.db1, self.db2) - for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: - doc = db.get_doc('doc') - doc.set_json(content) - db.put_doc(doc) - self.sync(self.db1, self.db2) - # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict - doc = self.db1.get_doc('doc') - rev1 = doc.rev - self.assertTrue(doc.has_conflicts) - # set db2 to have a doc of {} (same as db1 before the conflict) - doc = self.db2.get_doc('doc') - doc.set_json('{}') - self.db2.put_doc(doc) - rev2 = doc.rev - # sync it across - self.sync(self.db1, self.db2) - # tadaa! - doc = self.db1.get_doc('doc') - self.assertFalse(doc.has_conflicts) - vec1 = vectorclock.VectorClockRev(rev1) - vec2 = vectorclock.VectorClockRev(rev2) - vec3 = vectorclock.VectorClockRev(doc.rev) - self.assertTrue(vec3.is_newer(vec1)) - self.assertTrue(vec3.is_newer(vec2)) - # because the conflict is on the source, sync it another time - self.sync(self.db1, self.db2) - # make sure db2 now has the exact same thing - doc1 = self.db1.get_doc('doc') - self.assertGetEncryptedDoc( - self.db2, - doc1.doc_id, doc1.rev, doc1.get_json(), False) - - def test_sync_autoresolves_moar_backwards(self): - # here we would test that when a database that has a conflicted - # document is the target of a sync, and the source database has a - # revision of the conflicted document that is newer than the target - # database's, and that source's database's document's content is the - # same as the target's document's conflict's, the target's document's - # conflict gets autoresolved, and the document's revision bumped. - # - # Despite that, in Soledad we suppose that the server never syncs, so - # it never has conflicted documents. Also, if it had, convergence - # would not be possible by checking document's contents because they - # would be encrypted in server. - # - # Therefore we suppress this test. - pass - - def test_sync_autoresolves_moar_backwards_three(self): - # here we would test that when a database that has a conflicted - # document is the target of a sync, and the source database has a - # revision of the conflicted document that is newer than the target - # database's, and that source's database's document's content is the - # same as the target's document's conflict's, the target's document's - # conflict gets autoresolved, and the document's revision bumped. - # - # We use the same reasoning from the last test to suppress this one. - pass - - def test_sync_pulling_doesnt_update_other_if_changed(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db2.create_doc_from_json(tests.simple_doc) - # After the local side has sent its list of docs, before we start - # receiving the "targets" response, we update the local database with a - # new record. - # When we finish synchronizing, we can notice that something locally - # was updated, and we cannot tell c2 our new updated generation - - def before_get_docs(state): - if state != 'before get_docs': - return - self.db1.create_doc_from_json(tests.simple_doc) - - self.assertEqual(0, self.sync(self.db1, self.db2, - trace_hook=before_get_docs)) - self.assertLastExchangeLog(self.db2, - {'receive': - {'docs': [], 'last_known_gen': 0}, - 'return': - {'docs': [(doc.doc_id, doc.rev)], - 'last_gen': 1}}) - self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) - # c2 should not have gotten a '_record_sync_info' call, because the - # local database had been updated more than just by the messages - # returned from c2. - self.assertEqual( - (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) - - def test_sync_doesnt_update_other_if_nothing_pulled(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc) - - def no_record_sync_info(state): - if state != 'record_sync_info': - return - self.fail('SyncTarget.record_sync_info was called') - self.assertEqual(1, self.sync(self.db1, self.db2, - trace_hook_shallow=no_record_sync_info)) - self.assertEqual( - 1, - self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)[0]) - - def test_sync_ignores_convergence(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'both') - doc = self.db1.create_doc_from_json(tests.simple_doc) - self.db3 = self.create_database('test3', 'target') - self.assertEqual(1, self.sync(self.db1, self.db3)) - self.assertEqual(0, self.sync(self.db2, self.db3)) - self.assertEqual(1, self.sync(self.db1, self.db2)) - self.assertLastExchangeLog(self.db2, - {'receive': - {'docs': [(doc.doc_id, doc.rev)], - 'source_uid': 'test1', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': {'docs': [], 'last_gen': 1}}) - - def test_sync_ignores_superseded(self): - self.db1 = self.create_database('test1', 'both') - self.db2 = self.create_database('test2', 'both') - doc = self.db1.create_doc_from_json(tests.simple_doc) - doc_rev1 = doc.rev - self.db3 = self.create_database('test3', 'target') - self.sync(self.db1, self.db3) - self.sync(self.db2, self.db3) - new_content = '{"key": "altval"}' - doc.set_json(new_content) - self.db1.put_doc(doc) - doc_rev2 = doc.rev - self.sync(self.db2, self.db1) - self.assertLastExchangeLog(self.db1, - {'receive': - {'docs': [(doc.doc_id, doc_rev1)], - 'source_uid': 'test2', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': - {'docs': [(doc.doc_id, doc_rev2)], - 'last_gen': 2}}) - self.assertGetDoc(self.db1, doc.doc_id, doc_rev2, new_content, False) - - def test_sync_sees_remote_conflicted(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc1 = self.db1.create_doc_from_json(tests.simple_doc) - doc_id = doc1.doc_id - doc1_rev = doc1.rev - self.db1.create_index('test-idx', 'key') - new_doc = '{"key": "altval"}' - doc2 = self.db2.create_doc_from_json(new_doc, doc_id=doc_id) - doc2_rev = doc2.rev - self.assertTransactionLog([doc1.doc_id], self.db1) - self.sync(self.db1, self.db2) - self.assertLastExchangeLog(self.db2, - {'receive': - {'docs': [(doc_id, doc1_rev)], - 'source_uid': 'test1', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': - {'docs': [(doc_id, doc2_rev)], - 'last_gen': 1}}) - self.assertTransactionLog([doc_id, doc_id], self.db1) - self.assertGetDoc(self.db1, doc_id, doc2_rev, new_doc, True) - self.assertGetDoc(self.db2, doc_id, doc2_rev, new_doc, False) - from_idx = self.db1.get_from_index('test-idx', 'altval')[0] - self.assertEqual(doc2.doc_id, from_idx.doc_id) - self.assertEqual(doc2.rev, from_idx.rev) - self.assertTrue(from_idx.has_conflicts) - self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) - - def test_sync_sees_remote_delete_conflicted(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc1 = self.db1.create_doc_from_json(tests.simple_doc) - doc_id = doc1.doc_id - self.db1.create_index('test-idx', 'key') - self.sync(self.db1, self.db2) - doc2 = self.make_document(doc1.doc_id, doc1.rev, doc1.get_json()) - new_doc = '{"key": "altval"}' - doc1.set_json(new_doc) - self.db1.put_doc(doc1) - self.db2.delete_doc(doc2) - self.assertTransactionLog([doc_id, doc_id], self.db1) - self.sync(self.db1, self.db2) - self.assertLastExchangeLog(self.db2, - {'receive': - {'docs': [(doc_id, doc1.rev)], - 'source_uid': 'test1', - 'source_gen': 2, 'last_known_gen': 1}, - 'return': {'docs': [(doc_id, doc2.rev)], - 'last_gen': 2}}) - self.assertTransactionLog([doc_id, doc_id, doc_id], self.db1) - self.assertGetDocIncludeDeleted(self.db1, doc_id, doc2.rev, None, True) - self.assertGetDocIncludeDeleted( - self.db2, doc_id, doc2.rev, None, False) - self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) - - def test_sync_local_race_conflicted(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db1.create_doc_from_json(tests.simple_doc) - doc_id = doc.doc_id - doc1_rev = doc.rev - self.db1.create_index('test-idx', 'key') - self.sync(self.db1, self.db2) - content1 = '{"key": "localval"}' - content2 = '{"key": "altval"}' - doc.set_json(content2) - self.db2.put_doc(doc) - doc2_rev2 = doc.rev - triggered = [] - - def after_whatschanged(state): - if state != 'after whats_changed': - return - triggered.append(True) - doc = self.make_document(doc_id, doc1_rev, content1) - self.db1.put_doc(doc) - - self.sync(self.db1, self.db2, trace_hook=after_whatschanged) - self.assertEqual([True], triggered) - self.assertGetDoc(self.db1, doc_id, doc2_rev2, content2, True) - from_idx = self.db1.get_from_index('test-idx', 'altval')[0] - self.assertEqual(doc.doc_id, from_idx.doc_id) - self.assertEqual(doc.rev, from_idx.rev) - self.assertTrue(from_idx.has_conflicts) - self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) - self.assertEqual([], self.db1.get_from_index('test-idx', 'localval')) - - def test_sync_propagates_deletes(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'both') - doc1 = self.db1.create_doc_from_json(tests.simple_doc) - doc_id = doc1.doc_id - self.db1.create_index('test-idx', 'key') - self.sync(self.db1, self.db2) - self.db2.create_index('test-idx', 'key') - self.db3 = self.create_database('test3', 'target') - self.sync(self.db1, self.db3) - self.db1.delete_doc(doc1) - deleted_rev = doc1.rev - self.sync(self.db1, self.db2) - self.assertLastExchangeLog(self.db2, - {'receive': - {'docs': [(doc_id, deleted_rev)], - 'source_uid': 'test1', - 'source_gen': 2, 'last_known_gen': 1}, - 'return': {'docs': [], 'last_gen': 2}}) - self.assertGetDocIncludeDeleted( - self.db1, doc_id, deleted_rev, None, False) - self.assertGetDocIncludeDeleted( - self.db2, doc_id, deleted_rev, None, False) - self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) - self.assertEqual([], self.db2.get_from_index('test-idx', 'value')) - self.sync(self.db2, self.db3) - self.assertLastExchangeLog(self.db3, - {'receive': - {'docs': [(doc_id, deleted_rev)], - 'source_uid': 'test2', - 'source_gen': 2, - 'last_known_gen': 0}, - 'return': - {'docs': [], 'last_gen': 2}}) - self.assertGetDocIncludeDeleted( - self.db3, doc_id, deleted_rev, None, False) - - def test_sync_propagates_deletes_2(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json('{"a": "1"}', doc_id='the-doc') - self.sync(self.db1, self.db2) - doc1_2 = self.db2.get_doc('the-doc') - self.db2.delete_doc(doc1_2) - self.sync(self.db1, self.db2) - self.assertGetDocIncludeDeleted( - self.db1, 'the-doc', doc1_2.rev, None, False) - - def test_sync_detects_identical_replica_uid(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test1', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') - self.assertRaises( - errors.InvalidReplicaUID, self.sync, self.db1, self.db2) - - def test_optional_sync_preserve_json(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - cont1 = '{ "a": 2 }' - cont2 = '{ "b":3}' - self.db1.create_doc_from_json(cont1, doc_id="1") - self.db2.create_doc_from_json(cont2, doc_id="2") - self.sync(self.db1, self.db2) - self.assertEqual(cont1, self.db2.get_doc("1").get_json()) - self.assertEqual(cont2, self.db1.get_doc("2").get_json()) - - def test_sync_propagates_resolution(self): - """ - Test if synchronization propagates resolution. - - This test was adapted to decrypt remote content before assert. - """ - self.db1 = self.create_database('test1', 'both') - self.db2 = self.create_database('test2', 'both') - doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') - db3 = self.create_database('test3', 'both') - self.sync(self.db2, self.db1) - self.assertEqual( - self.db1._get_generation_info(), - self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)) - self.assertEqual( - self.db2._get_generation_info(), - self.db1._get_replica_gen_and_trans_id(self.db2._replica_uid)) - self.sync(db3, self.db1) - # update on 2 - doc2 = self.make_document('the-doc', doc1.rev, '{"a": 2}') - self.db2.put_doc(doc2) - self.sync(self.db2, db3) - self.assertEqual(db3.get_doc('the-doc').rev, doc2.rev) - # update on 1 - doc1.set_json('{"a": 3}') - self.db1.put_doc(doc1) - # conflicts - self.sync(self.db2, self.db1) - self.sync(db3, self.db1) - self.assertTrue(self.db2.get_doc('the-doc').has_conflicts) - self.assertTrue(db3.get_doc('the-doc').has_conflicts) - # resolve - conflicts = self.db2.get_doc_conflicts('the-doc') - doc4 = self.make_document('the-doc', None, '{"a": 4}') - revs = [doc.rev for doc in conflicts] - self.db2.resolve_doc(doc4, revs) - doc2 = self.db2.get_doc('the-doc') - self.assertEqual(doc4.get_json(), doc2.get_json()) - self.assertFalse(doc2.has_conflicts) - self.sync(self.db2, db3) - doc3 = db3.get_doc('the-doc') - if ENC_SCHEME_KEY in doc3.content: - _crypto = self._soledad._crypto - key = _crypto.doc_passphrase(doc3.doc_id) - secret = _crypto.secret - doc3.set_json(decrypt_doc_dict( - doc3.content, - doc3.doc_id, doc3.rev, key, secret)) - self.assertEqual(doc4.get_json(), doc3.get_json()) - self.assertFalse(doc3.has_conflicts) - self.db1.close() - self.db2.close() - db3.close() - - def test_sync_puts_changes(self): - """ - Test if sync puts changes in remote replica. - - This test was adapted to decrypt remote content before assert. - """ - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db1.create_doc_from_json(tests.simple_doc) - self.assertEqual(1, self.sync(self.db1, self.db2)) - self.assertGetEncryptedDoc( - self.db2, doc.doc_id, doc.rev, tests.simple_doc, False) - self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) - self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [(doc.doc_id, doc.rev)], - 'source_uid': 'test1', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': {'docs': [], 'last_gen': 1}}) - - def test_sync_pulls_changes(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db2.create_doc_from_json(tests.simple_doc) - self.db1.create_index('test-idx', 'key') - self.assertEqual(0, self.sync(self.db1, self.db2)) - self.assertGetDoc(self.db1, doc.doc_id, doc.rev, - tests.simple_doc, False) - self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) - self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) - self.assertLastExchangeLog(self.db2, - {'receive': - {'docs': [], 'last_known_gen': 0}, - 'return': - {'docs': [(doc.doc_id, doc.rev)], - 'last_gen': 1}}) - self.assertEqual([doc], self.db1.get_from_index('test-idx', 'value')) - - def test_sync_supersedes_conflicts(self): - self.db1 = self.create_database('test1', 'both') - self.db2 = self.create_database('test2', 'target') - self.db3 = self.create_database('test3', 'both') - doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') - self.db2.create_doc_from_json('{"b": 1}', doc_id='the-doc') - self.db3.create_doc_from_json('{"c": 1}', doc_id='the-doc') - self.sync(self.db3, self.db1) - self.assertEqual( - self.db1._get_generation_info(), - self.db3._get_replica_gen_and_trans_id(self.db1._replica_uid)) - self.assertEqual( - self.db3._get_generation_info(), - self.db1._get_replica_gen_and_trans_id(self.db3._replica_uid)) - self.sync(self.db3, self.db2) - self.assertEqual( - self.db2._get_generation_info(), - self.db3._get_replica_gen_and_trans_id(self.db2._replica_uid)) - self.assertEqual( - self.db3._get_generation_info(), - self.db2._get_replica_gen_and_trans_id(self.db3._replica_uid)) - self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) - doc1.set_json('{"a": 2}') - self.db1.put_doc(doc1) - self.sync(self.db3, self.db1) - # original doc1 should have been removed from conflicts - self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) - - def test_sync_stops_after_get_sync_info(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc) - self.sync(self.db1, self.db2) - - def put_hook(state): - self.fail("Tracehook triggered for %s" % (state,)) - - self.sync(self.db1, self.db2, trace_hook_shallow=put_hook) - - def test_sync_detects_rollback_in_source(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') - self.sync(self.db1, self.db2) - self.db1_copy = self.copy_database(self.db1) - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.sync(self.db1, self.db2) - self.assertRaises( - errors.InvalidGeneration, self.sync, self.db1_copy, self.db2) - - def test_sync_detects_rollback_in_target(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.db2_copy = self.copy_database(self.db2) - self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.sync(self.db1, self.db2) - self.assertRaises( - errors.InvalidGeneration, self.sync, self.db1, self.db2_copy) - - def test_sync_detects_diverged_source(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db3 = self.copy_database(self.db1) - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.db3.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.assertRaises( - errors.InvalidTransactionId, self.sync, self.db3, self.db2) - - def test_sync_detects_diverged_target(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db3 = self.copy_database(self.db2) - self.db3.create_doc_from_json(tests.nested_doc, doc_id="divergent") - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.assertRaises( - errors.InvalidTransactionId, self.sync, self.db1, self.db3) - - def test_sync_detects_rollback_and_divergence_in_source(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') - self.sync(self.db1, self.db2) - self.db1_copy = self.copy_database(self.db1) - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.sync(self.db1, self.db2) - self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.assertRaises( - errors.InvalidTransactionId, self.sync, self.db1_copy, self.db2) - - def test_sync_detects_rollback_and_divergence_in_target(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.db2_copy = self.copy_database(self.db2) - self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.sync(self.db1, self.db2) - self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.assertRaises( - errors.InvalidTransactionId, self.sync, self.db1, self.db2_copy) - - -def make_local_db_and_soledad_target( - test, path='test', - source_replica_uid=uuid4().hex): - test.startTwistedServer() - replica_uid = os.path.basename(path) - db = test.request_state._create_database(replica_uid) - sync_db = test._soledad._sync_db - sync_enc_pool = test._soledad._sync_enc_pool - st = soledad_sync_target( - test, db._dbname, - source_replica_uid=source_replica_uid, - sync_db=sync_db, - sync_enc_pool=sync_enc_pool) - return db, st - -target_scenarios = [ - ('leap', { - 'create_db_and_target': make_local_db_and_soledad_target, - 'make_app_with_state': make_soledad_app, - 'do_sync': sync_via_synchronizer_and_soledad}), -] - - -class SQLCipherSyncTargetTests(SoledadDatabaseSyncTargetTests): - - # TODO: implement _set_trace_hook(_shallow) in SoledadHTTPSyncTarget so - # skipped tests can be succesfully executed. - - scenarios = (tests.multiply_scenarios(SQLCIPHER_SCENARIOS, - target_scenarios)) - - whitebox = False diff --git a/common/src/leap/soledad/common/tests/test_sync.py b/common/src/leap/soledad/common/tests/test_sync.py deleted file mode 100644 index cc18d387..00000000 --- a/common/src/leap/soledad/common/tests/test_sync.py +++ /dev/null @@ -1,216 +0,0 @@ -# -*- coding: utf-8 -*- -# test_sync.py -# Copyright (C) 2013, 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -import json -import tempfile -import threading -import time - -from urlparse import urljoin -from twisted.internet import defer - -from testscenarios import TestWithScenarios - -from leap.soledad.common import couch -from leap.soledad.client import sync - -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests.u1db_tests import TestCaseWithServer -from leap.soledad.common.tests.u1db_tests import simple_doc -from leap.soledad.common.tests.util import make_token_soledad_app -from leap.soledad.common.tests.util import make_soledad_document_for_test -from leap.soledad.common.tests.util import soledad_sync_target -from leap.soledad.common.tests.util import BaseSoledadTest -from leap.soledad.common.tests.util import SoledadWithCouchServerMixin -from leap.soledad.common.tests.test_couch import CouchDBTestCase - - -class InterruptableSyncTestCase( - BaseSoledadTest, CouchDBTestCase, TestCaseWithServer): - - """ - Tests for encrypted sync using Soledad server backed by a couch database. - """ - - @staticmethod - def make_app_with_state(state): - return make_token_soledad_app(state) - - make_document_for_test = make_soledad_document_for_test - - sync_target = soledad_sync_target - - def make_app(self): - self.request_state = couch.CouchServerState(self.couch_url) - return self.make_app_with_state(self.request_state) - - def setUp(self): - TestCaseWithServer.setUp(self) - CouchDBTestCase.setUp(self) - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - - def tearDown(self): - CouchDBTestCase.tearDown(self) - TestCaseWithServer.tearDown(self) - - def test_interruptable_sync(self): - """ - Test if Soledad can sync many smallfiles. - """ - - self.skipTest("Sync is currently not interruptable.") - - class _SyncInterruptor(threading.Thread): - - """ - A thread meant to interrupt the sync process. - """ - - def __init__(self, soledad, couchdb): - self._soledad = soledad - self._couchdb = couchdb - threading.Thread.__init__(self) - - def run(self): - while db._get_generation() < 2: - # print "WAITING %d" % db._get_generation() - time.sleep(0.1) - self._soledad.stop_sync() - time.sleep(1) - - number_of_docs = 10 - self.startServer() - - # instantiate soledad and create a document - sol = self._soledad_instance( - user='user-uuid', server_url=self.getURL()) - - # ensure remote db exists before syncing - db = couch.CouchDatabase.open_database( - urljoin(self.couch_url, 'user-user-uuid'), - create=True, - ensure_ddocs=True) - - # create interruptor thread - t = _SyncInterruptor(sol, db) - t.start() - - d = sol.get_all_docs() - d.addCallback(lambda results: self.assertEqual([], results[1])) - - def _create_docs(results): - # create many small files - deferreds = [] - for i in range(0, number_of_docs): - deferreds.append(sol.create_doc(json.loads(simple_doc))) - return defer.DeferredList(deferreds) - - # sync with server - d.addCallback(_create_docs) - d.addCallback(lambda _: sol.get_all_docs()) - d.addCallback( - lambda results: self.assertEqual(number_of_docs, len(results[1]))) - d.addCallback(lambda _: sol.sync()) - d.addCallback(lambda _: t.join()) - d.addCallback(lambda _: db.get_all_docs()) - d.addCallback( - lambda results: self.assertNotEqual( - number_of_docs, len(results[1]))) - d.addCallback(lambda _: sol.sync()) - d.addCallback(lambda _: db.get_all_docs()) - d.addCallback( - lambda results: self.assertEqual(number_of_docs, len(results[1]))) - - def _tear_down(results): - db.delete_database() - db.close() - sol.close() - - d.addCallback(_tear_down) - return d - - -class TestSoledadDbSync( - TestWithScenarios, - SoledadWithCouchServerMixin, - tests.TestCaseWithServer): - - """ - Test db.sync remote sync shortcut - """ - - scenarios = [ - ('py-token-http', { - 'make_app_with_state': make_token_soledad_app, - 'make_database_for_test': tests.make_memory_database_for_test, - 'token': True - }), - ] - - oauth = False - token = False - - def setUp(self): - """ - Need to explicitely invoke inicialization on all bases. - """ - SoledadWithCouchServerMixin.setUp(self) - self.startTwistedServer() - self.db = self.make_database_for_test(self, 'test1') - self.db2 = self.request_state._create_database(replica_uid='test') - - def tearDown(self): - """ - Need to explicitely invoke destruction on all bases. - """ - SoledadWithCouchServerMixin.tearDown(self) - # tests.TestCaseWithServer.tearDown(self) - - def do_sync(self): - """ - Perform sync using SoledadSynchronizer, SoledadSyncTarget - and Token auth. - """ - target = soledad_sync_target( - self, self.db2._dbname, - source_replica_uid=self._soledad._dbpool.replica_uid) - self.addCleanup(target.close) - return sync.SoledadSynchronizer( - self.db, - target).sync(defer_decryption=False) - - @defer.inlineCallbacks - def test_db_sync(self): - """ - Test sync. - - Adapted to check for encrypted content. - """ - - doc1 = self.db.create_doc_from_json(tests.simple_doc) - doc2 = self.db2.create_doc_from_json(tests.nested_doc) - - local_gen_before_sync = yield self.do_sync() - gen, _, changes = self.db.whats_changed(local_gen_before_sync) - self.assertEqual(1, len(changes)) - self.assertEqual(doc2.doc_id, changes[0][0]) - self.assertEqual(1, gen - local_gen_before_sync) - self.assertGetEncryptedDoc( - self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) - self.assertGetEncryptedDoc( - self.db, doc2.doc_id, doc2.rev, tests.nested_doc, False) - - # TODO: add u1db.tests.test_sync.TestRemoteSyncIntegration diff --git a/common/src/leap/soledad/common/tests/test_sync_deferred.py b/common/src/leap/soledad/common/tests/test_sync_deferred.py deleted file mode 100644 index c62bd156..00000000 --- a/common/src/leap/soledad/common/tests/test_sync_deferred.py +++ /dev/null @@ -1,196 +0,0 @@ -# test_sync_deferred.py -# Copyright (C) 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Test Leap backend bits: sync with deferred encryption/decryption. -""" -import time -import os -import random -import string -import shutil - -from urlparse import urljoin - -from twisted.internet import defer - -from leap.soledad.common import couch - -from leap.soledad.client import sync -from leap.soledad.client.sqlcipher import SQLCipherOptions -from leap.soledad.client.sqlcipher import SQLCipherDatabase - -from testscenarios import TestWithScenarios - -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests.util import ADDRESS -from leap.soledad.common.tests.util import SoledadWithCouchServerMixin -from leap.soledad.common.tests.util import make_soledad_app -from leap.soledad.common.tests.util import soledad_sync_target - - -# Just to make clear how this test is different... :) -DEFER_DECRYPTION = True - -WAIT_STEP = 1 -MAX_WAIT = 10 -DBPASS = "pass" - - -class BaseSoledadDeferredEncTest(SoledadWithCouchServerMixin): - - """ - Another base class for testing the deferred encryption/decryption during - the syncs, using the intermediate database. - """ - defer_sync_encryption = True - - def setUp(self): - SoledadWithCouchServerMixin.setUp(self) - self.startTwistedServer() - # config info - self.db1_file = os.path.join(self.tempdir, "db1.u1db") - os.unlink(self.db1_file) - self.db_pass = DBPASS - self.email = ADDRESS - - # get a random prefix for each test, so we do not mess with - # concurrency during initialization and shutting down of - # each local db. - self.rand_prefix = ''.join( - map(lambda x: random.choice(string.ascii_letters), range(6))) - - # open test dbs: db1 will be the local sqlcipher db (which - # instantiates a syncdb). We use the self._soledad instance that was - # already created on some setUp method. - import binascii - tohex = binascii.b2a_hex - key = tohex(self._soledad.secrets.get_local_storage_key()) - sync_db_key = tohex(self._soledad.secrets.get_sync_db_key()) - dbpath = self._soledad._local_db_path - - self.opts = SQLCipherOptions( - dbpath, key, is_raw_key=True, create=False, - defer_encryption=True, sync_db_key=sync_db_key) - self.db1 = SQLCipherDatabase(self.opts) - - self.db2 = self.request_state._create_database('test') - - def tearDown(self): - # XXX should not access "private" attrs - shutil.rmtree(os.path.dirname(self._soledad._local_db_path)) - SoledadWithCouchServerMixin.tearDown(self) - - -class SyncTimeoutError(Exception): - - """ - Dummy exception to notify timeout during sync. - """ - pass - - -class TestSoledadDbSyncDeferredEncDecr( - TestWithScenarios, - BaseSoledadDeferredEncTest, - tests.TestCaseWithServer): - - """ - Test db.sync remote sync shortcut. - Case with deferred encryption and decryption: using the intermediate - syncdb. - """ - - scenarios = [ - ('http', { - 'make_app_with_state': make_soledad_app, - 'make_database_for_test': tests.make_memory_database_for_test, - }), - ] - - oauth = False - token = True - - def setUp(self): - """ - Need to explicitely invoke inicialization on all bases. - """ - BaseSoledadDeferredEncTest.setUp(self) - self.server = self.server_thread = None - self.syncer = None - - def tearDown(self): - """ - Need to explicitely invoke destruction on all bases. - """ - dbsyncer = getattr(self, 'dbsyncer', None) - if dbsyncer: - dbsyncer.close() - BaseSoledadDeferredEncTest.tearDown(self) - - def do_sync(self): - """ - Perform sync using SoledadSynchronizer, SoledadSyncTarget - and Token auth. - """ - replica_uid = self._soledad._dbpool.replica_uid - sync_db = self._soledad._sync_db - sync_enc_pool = self._soledad._sync_enc_pool - dbsyncer = self._soledad._dbsyncer # Soledad.sync uses the dbsyncer - - target = soledad_sync_target( - self, self.db2._dbname, - source_replica_uid=replica_uid, - sync_db=sync_db, - sync_enc_pool=sync_enc_pool) - self.addCleanup(target.close) - return sync.SoledadSynchronizer( - dbsyncer, - target).sync(defer_decryption=True) - - def wait_for_sync(self): - """ - Wait for sync to finish. - """ - wait = 0 - syncer = self.syncer - if syncer is not None: - while syncer.syncing: - time.sleep(WAIT_STEP) - wait += WAIT_STEP - if wait >= MAX_WAIT: - raise SyncTimeoutError - - @defer.inlineCallbacks - def test_db_sync(self): - """ - Test sync. - - Adapted to check for encrypted content. - """ - doc1 = self.db1.create_doc_from_json(tests.simple_doc) - doc2 = self.db2.create_doc_from_json(tests.nested_doc) - local_gen_before_sync = yield self.do_sync() - - gen, _, changes = self.db1.whats_changed(local_gen_before_sync) - self.assertEqual(1, len(changes)) - - self.assertEqual(doc2.doc_id, changes[0][0]) - self.assertEqual(1, gen - local_gen_before_sync) - - self.assertGetEncryptedDoc( - self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) - self.assertGetEncryptedDoc( - self.db1, doc2.doc_id, doc2.rev, tests.nested_doc, False) diff --git a/common/src/leap/soledad/common/tests/test_sync_mutex.py b/common/src/leap/soledad/common/tests/test_sync_mutex.py deleted file mode 100644 index 973a8587..00000000 --- a/common/src/leap/soledad/common/tests/test_sync_mutex.py +++ /dev/null @@ -1,135 +0,0 @@ -# -*- coding: utf-8 -*- -# test_sync_mutex.py -# Copyright (C) 2013, 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -Test that synchronization is a critical section and, as such, there might not -be two concurrent synchronization processes at the same time. -""" - - -import time -import uuid -import tempfile -import shutil - -from urlparse import urljoin - -from twisted.internet import defer - -from leap.soledad.client.sync import SoledadSynchronizer - -from leap.soledad.common.couch.state import CouchServerState -from leap.soledad.common.couch import CouchDatabase -from leap.soledad.common.tests.u1db_tests import TestCaseWithServer -from leap.soledad.common.tests.test_couch import CouchDBTestCase - -from leap.soledad.common.tests.util import BaseSoledadTest -from leap.soledad.common.tests.util import make_token_soledad_app -from leap.soledad.common.tests.util import make_soledad_document_for_test -from leap.soledad.common.tests.util import soledad_sync_target - - -# monkey-patch the soledad synchronizer so it stores start and finish times - -_old_sync = SoledadSynchronizer.sync - - -def _timed_sync(self, defer_decryption=True): - t = time.time() - - sync_id = uuid.uuid4() - - if not getattr(self.source, 'sync_times', False): - self.source.sync_times = {} - - self.source.sync_times[sync_id] = {'start': t} - - def _store_finish_time(passthrough): - t = time.time() - self.source.sync_times[sync_id]['end'] = t - return passthrough - - d = _old_sync(self, defer_decryption=defer_decryption) - d.addBoth(_store_finish_time) - return d - -SoledadSynchronizer.sync = _timed_sync - -# -- end of monkey-patching - - -class TestSyncMutex( - BaseSoledadTest, CouchDBTestCase, TestCaseWithServer): - - @staticmethod - def make_app_with_state(state): - return make_token_soledad_app(state) - - make_document_for_test = make_soledad_document_for_test - - sync_target = soledad_sync_target - - def make_app(self): - self.request_state = CouchServerState(self.couch_url) - return self.make_app_with_state(self.request_state) - - def setUp(self): - TestCaseWithServer.setUp(self) - CouchDBTestCase.setUp(self) - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.user = ('user-%s' % uuid.uuid4().hex) - - def tearDown(self): - CouchDBTestCase.tearDown(self) - TestCaseWithServer.tearDown(self) - shutil.rmtree(self.tempdir) - - def test_two_concurrent_syncs_do_not_overlap_no_docs(self): - self.startServer() - - # ensure remote db exists before syncing - db = CouchDatabase.open_database( - urljoin(self.couch_url, 'user-' + self.user), - create=True, - ensure_ddocs=True) - - sol = self._soledad_instance( - user=self.user, server_url=self.getURL()) - - d1 = sol.sync() - d2 = sol.sync() - - def _assert_syncs_do_not_overlap(thearg): - # recover sync times - sync_times = [] - for key in sol._dbsyncer.sync_times: - sync_times.append(sol._dbsyncer.sync_times[key]) - sync_times.sort(key=lambda s: s['start']) - - self.assertTrue( - (sync_times[0]['start'] < sync_times[0]['end'] and - sync_times[0]['end'] < sync_times[1]['start'] and - sync_times[1]['start'] < sync_times[1]['end'])) - - db.delete_database() - db.close() - sol.close() - - d = defer.gatherResults([d1, d2]) - d.addBoth(_assert_syncs_do_not_overlap) - return d diff --git a/common/src/leap/soledad/common/tests/test_sync_target.py b/common/src/leap/soledad/common/tests/test_sync_target.py deleted file mode 100644 index c9b705a3..00000000 --- a/common/src/leap/soledad/common/tests/test_sync_target.py +++ /dev/null @@ -1,956 +0,0 @@ -# -*- coding: utf-8 -*- -# test_sync_target.py -# Copyright (C) 2013, 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Test Leap backend bits: sync target -""" -import cStringIO -import os -import time -import json -import random -import string -import shutil -from uuid import uuid4 - -from testscenarios import TestWithScenarios -from twisted.internet import defer - -from leap.soledad.client import http_target as target -from leap.soledad.client import crypto -from leap.soledad.client.sqlcipher import SQLCipherU1DBSync -from leap.soledad.client.sqlcipher import SQLCipherOptions -from leap.soledad.client.sqlcipher import SQLCipherDatabase - -from leap.soledad.common import l2db - -from leap.soledad.common.document import SoledadDocument -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests.util import make_sqlcipher_database_for_test -from leap.soledad.common.tests.util import make_soledad_app -from leap.soledad.common.tests.util import make_token_soledad_app -from leap.soledad.common.tests.util import make_soledad_document_for_test -from leap.soledad.common.tests.util import soledad_sync_target -from leap.soledad.common.tests.util import SoledadWithCouchServerMixin -from leap.soledad.common.tests.util import ADDRESS - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_remote_sync_target`. -# ----------------------------------------------------------------------------- - -class TestSoledadParseReceivedDocResponse(SoledadWithCouchServerMixin): - - """ - Some tests had to be copied to this class so we can instantiate our own - target. - """ - - def setUp(self): - SoledadWithCouchServerMixin.setUp(self) - creds = {'token': { - 'uuid': 'user-uuid', - 'token': 'auth-token', - }} - self.target = target.SoledadHTTPSyncTarget( - self.couch_url, - uuid4().hex, - creds, - self._soledad._crypto, - None) - - def tearDown(self): - self.target.close() - SoledadWithCouchServerMixin.tearDown(self) - - def test_extra_comma(self): - """ - Test adapted to use encrypted content. - """ - doc = SoledadDocument('i', rev='r') - doc.content = {} - _crypto = self._soledad._crypto - key = _crypto.doc_passphrase(doc.doc_id) - secret = _crypto.secret - - enc_json = crypto.encrypt_docstr( - doc.get_json(), doc.doc_id, doc.rev, - key, secret) - - with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("[\r\n{},\r\n]") - - with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response( - ('[\r\n{},\r\n{"id": "i", "rev": "r", ' + - '"content": %s, "gen": 3, "trans_id": "T-sid"}' + - ',\r\n]') % json.dumps(enc_json)) - - def test_wrong_start(self): - with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("{}\r\n]") - - with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("\r\n{}\r\n]") - - with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("") - - def test_wrong_end(self): - with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("[\r\n{}") - - with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("[\r\n") - - def test_missing_comma(self): - with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response( - '[\r\n{}\r\n{"id": "i", "rev": "r", ' - '"content": "c", "gen": 3}\r\n]') - - def test_no_entries(self): - with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response("[\r\n]") - - def test_error_in_stream(self): - with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response( - '[\r\n{"new_generation": 0},' - '\r\n{"error": "unavailable"}\r\n') - - with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response( - '[\r\n{"error": "unavailable"}\r\n') - - with self.assertRaises(l2db.errors.BrokenSyncStream): - self.target._parse_received_doc_response('[\r\n{"error": "?"}\r\n') - -# -# functions for TestRemoteSyncTargets -# - - -def make_local_db_and_soledad_target( - test, path='test', - source_replica_uid=uuid4().hex): - test.startTwistedServer() - replica_uid = os.path.basename(path) - db = test.request_state._create_database(replica_uid) - sync_db = test._soledad._sync_db - sync_enc_pool = test._soledad._sync_enc_pool - st = soledad_sync_target( - test, db._dbname, - source_replica_uid=source_replica_uid, - sync_db=sync_db, - sync_enc_pool=sync_enc_pool) - return db, st - - -def make_local_db_and_token_soledad_target( - test, - source_replica_uid=uuid4().hex): - db, st = make_local_db_and_soledad_target( - test, path='test', - source_replica_uid=source_replica_uid) - st.set_token_credentials('user-uuid', 'auth-token') - return db, st - - -class TestSoledadSyncTarget( - TestWithScenarios, - SoledadWithCouchServerMixin, - tests.TestCaseWithServer): - - scenarios = [ - ('token_soledad', - {'make_app_with_state': make_token_soledad_app, - 'make_document_for_test': make_soledad_document_for_test, - 'create_db_and_target': make_local_db_and_token_soledad_target, - 'make_database_for_test': make_sqlcipher_database_for_test, - 'sync_target': soledad_sync_target}), - ] - - def getSyncTarget(self, path=None, source_replica_uid=uuid4().hex): - if self.port is None: - self.startTwistedServer() - sync_db = self._soledad._sync_db - sync_enc_pool = self._soledad._sync_enc_pool - if path is None: - path = self.db2._dbname - target = self.sync_target( - self, path, - source_replica_uid=source_replica_uid, - sync_db=sync_db, - sync_enc_pool=sync_enc_pool) - self.addCleanup(target.close) - return target - - def setUp(self): - TestWithScenarios.setUp(self) - SoledadWithCouchServerMixin.setUp(self) - self.startTwistedServer() - self.db1 = make_sqlcipher_database_for_test(self, 'test1') - self.db2 = self.request_state._create_database('test') - - def tearDown(self): - # db2, _ = self.request_state.ensure_database('test2') - self.delete_db(self.db2._dbname) - self.db1.close() - SoledadWithCouchServerMixin.tearDown(self) - TestWithScenarios.tearDown(self) - - @defer.inlineCallbacks - def test_sync_exchange_send(self): - """ - Test for sync exchanging send of document. - - This test was adapted to decrypt remote content before assert. - """ - db = self.db2 - remote_target = self.getSyncTarget() - other_docs = [] - - def receive_doc(doc, gen, trans_id): - other_docs.append((doc.doc_id, doc.rev, doc.get_json())) - - doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') - new_gen, trans_id = yield remote_target.sync_exchange( - [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=receive_doc, - defer_decryption=False) - self.assertEqual(1, new_gen) - self.assertGetEncryptedDoc( - db, 'doc-here', 'replica:1', '{"value": "here"}', False) - - @defer.inlineCallbacks - def test_sync_exchange_send_failure_and_retry_scenario(self): - """ - Test for sync exchange failure and retry. - - This test was adapted to decrypt remote content before assert. - """ - - def blackhole_getstderr(inst): - return cStringIO.StringIO() - - db = self.db2 - _put_doc_if_newer = db._put_doc_if_newer - trigger_ids = ['doc-here2'] - - def bomb_put_doc_if_newer(self, doc, save_conflict, - replica_uid=None, replica_gen=None, - replica_trans_id=None, number_of_docs=None, - doc_idx=None, sync_id=None): - if doc.doc_id in trigger_ids: - raise l2db.errors.U1DBError - return _put_doc_if_newer(doc, save_conflict=save_conflict, - replica_uid=replica_uid, - replica_gen=replica_gen, - replica_trans_id=replica_trans_id, - number_of_docs=number_of_docs, - doc_idx=doc_idx, sync_id=sync_id) - from leap.soledad.common.backend import SoledadBackend - self.patch( - SoledadBackend, '_put_doc_if_newer', bomb_put_doc_if_newer) - remote_target = self.getSyncTarget( - source_replica_uid='replica') - other_changes = [] - - def receive_doc(doc, gen, trans_id): - other_changes.append( - (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) - - doc1 = self.make_document('doc-here', 'replica:1', '{"value": "here"}') - doc2 = self.make_document('doc-here2', 'replica:1', - '{"value": "here2"}') - - with self.assertRaises(l2db.errors.U1DBError): - yield remote_target.sync_exchange( - [(doc1, 10, 'T-sid'), (doc2, 11, 'T-sud')], - 'replica', - last_known_generation=0, - last_known_trans_id=None, - insert_doc_cb=receive_doc, - defer_decryption=False) - - self.assertGetEncryptedDoc( - db, 'doc-here', 'replica:1', '{"value": "here"}', - False) - self.assertEqual( - (10, 'T-sid'), db._get_replica_gen_and_trans_id('replica')) - self.assertEqual([], other_changes) - # retry - trigger_ids = [] - new_gen, trans_id = yield remote_target.sync_exchange( - [(doc2, 11, 'T-sud')], 'replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=receive_doc, - defer_decryption=False) - self.assertGetEncryptedDoc( - db, 'doc-here2', 'replica:1', '{"value": "here2"}', - False) - self.assertEqual( - (11, 'T-sud'), db._get_replica_gen_and_trans_id('replica')) - self.assertEqual(2, new_gen) - self.assertEqual( - ('doc-here', 'replica:1', '{"value": "here"}', 1), - other_changes[0][:-1]) - - @defer.inlineCallbacks - def test_sync_exchange_send_ensure_callback(self): - """ - Test for sync exchange failure and retry. - - This test was adapted to decrypt remote content before assert. - """ - remote_target = self.getSyncTarget() - other_docs = [] - replica_uid_box = [] - - def receive_doc(doc, gen, trans_id): - other_docs.append((doc.doc_id, doc.rev, doc.get_json())) - - def ensure_cb(replica_uid): - replica_uid_box.append(replica_uid) - - doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') - new_gen, trans_id = yield remote_target.sync_exchange( - [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=receive_doc, - ensure_callback=ensure_cb, defer_decryption=False) - self.assertEqual(1, new_gen) - db = self.db2 - self.assertEqual(1, len(replica_uid_box)) - self.assertEqual(db._replica_uid, replica_uid_box[0]) - self.assertGetEncryptedDoc( - db, 'doc-here', 'replica:1', '{"value": "here"}', False) - - def test_sync_exchange_in_stream_error(self): - self.skipTest("bypass this test because our sync_exchange process " - "does not return u1db error 503 \"unavailable\" for " - "now") - - @defer.inlineCallbacks - def test_get_sync_info(self): - db = self.db2 - db._set_replica_gen_and_trans_id('other-id', 1, 'T-transid') - remote_target = self.getSyncTarget( - source_replica_uid='other-id') - sync_info = yield remote_target.get_sync_info('other-id') - self.assertEqual( - ('test', 0, '', 1, 'T-transid'), - sync_info) - - @defer.inlineCallbacks - def test_record_sync_info(self): - remote_target = self.getSyncTarget( - source_replica_uid='other-id') - yield remote_target.record_sync_info('other-id', 2, 'T-transid') - self.assertEqual((2, 'T-transid'), - self.db2._get_replica_gen_and_trans_id('other-id')) - - @defer.inlineCallbacks - def test_sync_exchange_receive(self): - db = self.db2 - doc = db.create_doc_from_json('{"value": "there"}') - remote_target = self.getSyncTarget() - other_changes = [] - - def receive_doc(doc, gen, trans_id): - other_changes.append( - (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) - - new_gen, trans_id = yield remote_target.sync_exchange( - [], 'replica', last_known_generation=0, last_known_trans_id=None, - insert_doc_cb=receive_doc) - self.assertEqual(1, new_gen) - self.assertEqual( - (doc.doc_id, doc.rev, '{"value": "there"}', 1), - other_changes[0][:-1]) - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_sync`. -# ----------------------------------------------------------------------------- - -target_scenarios = [ - ('mem,token_soledad', - {'create_db_and_target': make_local_db_and_token_soledad_target, - 'make_app_with_state': make_soledad_app, - 'make_database_for_test': tests.make_memory_database_for_test, - 'copy_database_for_test': tests.copy_memory_database_for_test, - 'make_document_for_test': tests.make_document_for_test}) -] - - -class SoledadDatabaseSyncTargetTests( - TestWithScenarios, - SoledadWithCouchServerMixin, - tests.DatabaseBaseTests, - tests.TestCaseWithServer): - """ - Adaptation of u1db.tests.test_sync.DatabaseSyncTargetTests. - """ - - # TODO: implement _set_trace_hook(_shallow) in SoledadHTTPSyncTarget so - # skipped tests can be succesfully executed. - - scenarios = target_scenarios - - whitebox = False - - def setUp(self): - tests.TestCaseWithServer.setUp(self) - self.other_changes = [] - SoledadWithCouchServerMixin.setUp(self) - self.db, self.st = make_local_db_and_soledad_target(self) - - def tearDown(self): - self.db.close() - self.st.close() - tests.TestCaseWithServer.tearDown(self) - SoledadWithCouchServerMixin.tearDown(self) - - def set_trace_hook(self, callback, shallow=False): - setter = (self.st._set_trace_hook if not shallow else - self.st._set_trace_hook_shallow) - try: - setter(callback) - except NotImplementedError: - self.skipTest("%s does not implement _set_trace_hook" - % (self.st.__class__.__name__,)) - - @defer.inlineCallbacks - def test_sync_exchange(self): - """ - Test sync exchange. - - This test was adapted to decrypt remote content before assert. - """ - docs_by_gen = [ - (self.make_document('doc-id', 'replica:1', tests.simple_doc), 10, - 'T-sid')] - new_gen, trans_id = yield self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc, - defer_decryption=False) - self.assertGetEncryptedDoc( - self.db, 'doc-id', 'replica:1', tests.simple_doc, False) - self.assertTransactionLog(['doc-id'], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 1, last_trans_id), - (self.other_changes, new_gen, last_trans_id)) - sync_info = yield self.st.get_sync_info('replica') - self.assertEqual(10, sync_info[3]) - - @defer.inlineCallbacks - def test_sync_exchange_push_many(self): - """ - Test sync exchange. - - This test was adapted to decrypt remote content before assert. - """ - docs_by_gen = [ - (self.make_document( - 'doc-id', 'replica:1', tests.simple_doc), 10, 'T-1'), - (self.make_document( - 'doc-id2', 'replica:1', tests.nested_doc), 11, 'T-2')] - new_gen, trans_id = yield self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc, - defer_decryption=False) - self.assertGetEncryptedDoc( - self.db, 'doc-id', 'replica:1', tests.simple_doc, False) - self.assertGetEncryptedDoc( - self.db, 'doc-id2', 'replica:1', tests.nested_doc, False) - self.assertTransactionLog(['doc-id', 'doc-id2'], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 2, last_trans_id), - (self.other_changes, new_gen, trans_id)) - sync_info = yield self.st.get_sync_info('replica') - self.assertEqual(11, sync_info[3]) - - @defer.inlineCallbacks - def test_sync_exchange_returns_many_new_docs(self): - """ - Test sync exchange. - - This test was adapted to avoid JSON serialization comparison as local - and remote representations might differ. It looks directly at the - doc's contents instead. - """ - doc = self.db.create_doc_from_json(tests.simple_doc) - doc2 = self.db.create_doc_from_json(tests.nested_doc) - self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) - new_gen, _ = yield self.st.sync_exchange( - [], 'other-replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc, - defer_decryption=False) - self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) - self.assertEqual(2, new_gen) - self.assertEqual( - [(doc.doc_id, doc.rev, 1), - (doc2.doc_id, doc2.rev, 2)], - [c[:-3] + c[-2:-1] for c in self.other_changes]) - self.assertEqual( - json.loads(tests.simple_doc), - json.loads(self.other_changes[0][2])) - self.assertEqual( - json.loads(tests.nested_doc), - json.loads(self.other_changes[1][2])) - if self.whitebox: - self.assertEqual( - self.db._last_exchange_log['return'], - {'last_gen': 2, 'docs': - [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]}) - - def receive_doc(self, doc, gen, trans_id): - self.other_changes.append( - (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) - - def test_get_sync_target(self): - self.assertIsNot(None, self.st) - - @defer.inlineCallbacks - def test_get_sync_info(self): - sync_info = yield self.st.get_sync_info('other') - self.assertEqual( - ('test', 0, '', 0, ''), sync_info) - - @defer.inlineCallbacks - def test_create_doc_updates_sync_info(self): - sync_info = yield self.st.get_sync_info('other') - self.assertEqual( - ('test', 0, '', 0, ''), sync_info) - self.db.create_doc_from_json(tests.simple_doc) - sync_info = yield self.st.get_sync_info('other') - self.assertEqual(1, sync_info[1]) - - @defer.inlineCallbacks - def test_record_sync_info(self): - yield self.st.record_sync_info('replica', 10, 'T-transid') - sync_info = yield self.st.get_sync_info('replica') - self.assertEqual( - ('test', 0, '', 10, 'T-transid'), sync_info) - - @defer.inlineCallbacks - def test_sync_exchange_deleted(self): - doc = self.db.create_doc_from_json('{}') - edit_rev = 'replica:1|' + doc.rev - docs_by_gen = [ - (self.make_document(doc.doc_id, edit_rev, None), 10, 'T-sid')] - new_gen, trans_id = yield self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc) - self.assertGetDocIncludeDeleted( - self.db, doc.doc_id, edit_rev, None, False) - self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 2, last_trans_id), - (self.other_changes, new_gen, trans_id)) - sync_info = yield self.st.get_sync_info('replica') - self.assertEqual(10, sync_info[3]) - - @defer.inlineCallbacks - def test_sync_exchange_refuses_conflicts(self): - doc = self.db.create_doc_from_json(tests.simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_doc = '{"key": "altval"}' - docs_by_gen = [ - (self.make_document(doc.doc_id, 'replica:1', new_doc), 10, - 'T-sid')] - new_gen, _ = yield self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id], self.db) - self.assertEqual( - (doc.doc_id, doc.rev, tests.simple_doc, 1), - self.other_changes[0][:-1]) - self.assertEqual(1, new_gen) - if self.whitebox: - self.assertEqual(self.db._last_exchange_log['return'], - {'last_gen': 1, 'docs': [(doc.doc_id, doc.rev)]}) - - @defer.inlineCallbacks - def test_sync_exchange_ignores_convergence(self): - doc = self.db.create_doc_from_json(tests.simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - gen, txid = self.db._get_generation_info() - docs_by_gen = [ - (self.make_document(doc.doc_id, doc.rev, tests.simple_doc), - 10, 'T-sid')] - new_gen, _ = yield self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=gen, - last_known_trans_id=txid, insert_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id], self.db) - self.assertEqual(([], 1), (self.other_changes, new_gen)) - - @defer.inlineCallbacks - def test_sync_exchange_returns_new_docs(self): - doc = self.db.create_doc_from_json(tests.simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_gen, _ = yield self.st.sync_exchange( - [], 'other-replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id], self.db) - self.assertEqual( - (doc.doc_id, doc.rev, tests.simple_doc, 1), - self.other_changes[0][:-1]) - self.assertEqual(1, new_gen) - if self.whitebox: - self.assertEqual(self.db._last_exchange_log['return'], - {'last_gen': 1, 'docs': [(doc.doc_id, doc.rev)]}) - - @defer.inlineCallbacks - def test_sync_exchange_returns_deleted_docs(self): - doc = self.db.create_doc_from_json(tests.simple_doc) - self.db.delete_doc(doc) - self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) - new_gen, _ = yield self.st.sync_exchange( - [], 'other-replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) - self.assertEqual( - (doc.doc_id, doc.rev, None, 2), self.other_changes[0][:-1]) - self.assertEqual(2, new_gen) - if self.whitebox: - self.assertEqual(self.db._last_exchange_log['return'], - {'last_gen': 2, 'docs': [(doc.doc_id, doc.rev)]}) - - @defer.inlineCallbacks - def test_sync_exchange_getting_newer_docs(self): - doc = self.db.create_doc_from_json(tests.simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_doc = '{"key": "altval"}' - docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, - 'T-sid')] - new_gen, _ = yield self.st.sync_exchange( - docs_by_gen, 'other-replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) - self.assertEqual(([], 2), (self.other_changes, new_gen)) - - @defer.inlineCallbacks - def test_sync_exchange_with_concurrent_updates_of_synced_doc(self): - expected = [] - - def before_whatschanged_cb(state): - if state != 'before whats_changed': - return - cont = '{"key": "cuncurrent"}' - conc_rev = self.db.put_doc( - self.make_document(doc.doc_id, 'test:1|z:2', cont)) - expected.append((doc.doc_id, conc_rev, cont, 3)) - - self.set_trace_hook(before_whatschanged_cb) - doc = self.db.create_doc_from_json(tests.simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_doc = '{"key": "altval"}' - docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, - 'T-sid')] - new_gen, _ = yield self.st.sync_exchange( - docs_by_gen, 'other-replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc) - self.assertEqual(expected, [c[:-1] for c in self.other_changes]) - self.assertEqual(3, new_gen) - - @defer.inlineCallbacks - def test_sync_exchange_with_concurrent_updates(self): - - def after_whatschanged_cb(state): - if state != 'after whats_changed': - return - self.db.create_doc_from_json('{"new": "doc"}') - - self.set_trace_hook(after_whatschanged_cb) - doc = self.db.create_doc_from_json(tests.simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_doc = '{"key": "altval"}' - docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, - 'T-sid')] - new_gen, _ = yield self.st.sync_exchange( - docs_by_gen, 'other-replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc) - self.assertEqual(([], 2), (self.other_changes, new_gen)) - - @defer.inlineCallbacks - def test_sync_exchange_converged_handling(self): - doc = self.db.create_doc_from_json(tests.simple_doc) - docs_by_gen = [ - (self.make_document('new', 'other:1', '{}'), 4, 'T-foo'), - (self.make_document(doc.doc_id, doc.rev, doc.get_json()), 5, - 'T-bar')] - new_gen, _ = yield self.st.sync_exchange( - docs_by_gen, 'other-replica', last_known_generation=0, - last_known_trans_id=None, insert_doc_cb=self.receive_doc) - self.assertEqual(([], 2), (self.other_changes, new_gen)) - - @defer.inlineCallbacks - def test_sync_exchange_detect_incomplete_exchange(self): - def before_get_docs_explode(state): - if state != 'before get_docs': - return - raise l2db.errors.U1DBError("fail") - self.set_trace_hook(before_get_docs_explode) - # suppress traceback printing in the wsgiref server - # self.patch(simple_server.ServerHandler, - # 'log_exception', lambda h, exc_info: None) - doc = self.db.create_doc_from_json(tests.simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - self.assertRaises( - (l2db.errors.U1DBError, l2db.errors.BrokenSyncStream), - self.st.sync_exchange, [], 'other-replica', - last_known_generation=0, last_known_trans_id=None, - insert_doc_cb=self.receive_doc) - - @defer.inlineCallbacks - def test_sync_exchange_doc_ids(self): - sync_exchange_doc_ids = getattr(self.st, 'sync_exchange_doc_ids', None) - if sync_exchange_doc_ids is None: - self.skipTest("sync_exchange_doc_ids not implemented") - db2 = self.create_database('test2') - doc = db2.create_doc_from_json(tests.simple_doc) - new_gen, trans_id = yield sync_exchange_doc_ids( - db2, [(doc.doc_id, 10, 'T-sid')], 0, None, - insert_doc_cb=self.receive_doc) - self.assertGetDoc(self.db, doc.doc_id, doc.rev, - tests.simple_doc, False) - self.assertTransactionLog([doc.doc_id], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 1, last_trans_id), - (self.other_changes, new_gen, trans_id)) - self.assertEqual(10, self.st.get_sync_info(db2._replica_uid)[3]) - - @defer.inlineCallbacks - def test__set_trace_hook(self): - called = [] - - def cb(state): - called.append(state) - - self.set_trace_hook(cb) - yield self.st.sync_exchange([], 'replica', 0, None, self.receive_doc) - yield self.st.record_sync_info('replica', 0, 'T-sid') - self.assertEqual(['before whats_changed', - 'after whats_changed', - 'before get_docs', - 'record_sync_info', - ], - called) - - @defer.inlineCallbacks - def test__set_trace_hook_shallow(self): - if (self.st._set_trace_hook_shallow == self.st._set_trace_hook or - self.st._set_trace_hook_shallow.im_func == - target.SoledadHTTPSyncTarget._set_trace_hook_shallow.im_func): - # shallow same as full - expected = ['before whats_changed', - 'after whats_changed', - 'before get_docs', - 'record_sync_info', - ] - else: - expected = ['sync_exchange', 'record_sync_info'] - - called = [] - - def cb(state): - called.append(state) - - self.set_trace_hook(cb, shallow=True) - yield self.st.sync_exchange([], 'replica', 0, None, self.receive_doc) - yield self.st.record_sync_info('replica', 0, 'T-sid') - self.assertEqual(expected, called) - - -# Just to make clear how this test is different... :) -DEFER_DECRYPTION = False - -WAIT_STEP = 1 -MAX_WAIT = 10 -DBPASS = "pass" - - -class SyncTimeoutError(Exception): - - """ - Dummy exception to notify timeout during sync. - """ - pass - - -class TestSoledadDbSync( - TestWithScenarios, - SoledadWithCouchServerMixin, - tests.TestCaseWithServer): - - """Test db.sync remote sync shortcut""" - - scenarios = [ - ('py-token-http', { - 'create_db_and_target': make_local_db_and_token_soledad_target, - 'make_app_with_state': make_token_soledad_app, - 'make_database_for_test': make_sqlcipher_database_for_test, - 'token': True - }), - ] - - oauth = False - token = False - - def setUp(self): - """ - Need to explicitely invoke inicialization on all bases. - """ - SoledadWithCouchServerMixin.setUp(self) - self.server = self.server_thread = None - self.startTwistedServer() - self.syncer = None - - # config info - self.db1_file = os.path.join(self.tempdir, "db1.u1db") - os.unlink(self.db1_file) - self.db_pass = DBPASS - self.email = ADDRESS - - # get a random prefix for each test, so we do not mess with - # concurrency during initialization and shutting down of - # each local db. - self.rand_prefix = ''.join( - map(lambda x: random.choice(string.ascii_letters), range(6))) - - # open test dbs: db1 will be the local sqlcipher db (which - # instantiates a syncdb). We use the self._soledad instance that was - # already created on some setUp method. - import binascii - tohex = binascii.b2a_hex - key = tohex(self._soledad.secrets.get_local_storage_key()) - sync_db_key = tohex(self._soledad.secrets.get_sync_db_key()) - dbpath = self._soledad._local_db_path - - self.opts = SQLCipherOptions( - dbpath, key, is_raw_key=True, create=False, - defer_encryption=True, sync_db_key=sync_db_key) - self.db1 = SQLCipherDatabase(self.opts) - - self.db2 = self.request_state._create_database(replica_uid='test') - - def tearDown(self): - """ - Need to explicitely invoke destruction on all bases. - """ - dbsyncer = getattr(self, 'dbsyncer', None) - if dbsyncer: - dbsyncer.close() - self.db1.close() - self.db2.close() - self._soledad.close() - - # XXX should not access "private" attrs - shutil.rmtree(os.path.dirname(self._soledad._local_db_path)) - SoledadWithCouchServerMixin.tearDown(self) - - def do_sync(self, target_name): - """ - Perform sync using SoledadSynchronizer, SoledadSyncTarget - and Token auth. - """ - if self.token: - creds = {'token': { - 'uuid': 'user-uuid', - 'token': 'auth-token', - }} - target_url = self.getURL(self.db2._dbname) - - # get a u1db syncer - crypto = self._soledad._crypto - replica_uid = self.db1._replica_uid - dbsyncer = SQLCipherU1DBSync( - self.opts, - crypto, - replica_uid, - None, - defer_encryption=True) - self.dbsyncer = dbsyncer - return dbsyncer.sync(target_url, - creds=creds, - defer_decryption=DEFER_DECRYPTION) - else: - return self._do_sync(self, target_name) - - def _do_sync(self, target_name): - if self.oauth: - path = '~/' + target_name - extra = dict(creds={'oauth': { - 'consumer_key': tests.consumer1.key, - 'consumer_secret': tests.consumer1.secret, - 'token_key': tests.token1.key, - 'token_secret': tests.token1.secret, - }}) - else: - path = target_name - extra = {} - target_url = self.getURL(path) - return self.db.sync(target_url, **extra) - - def wait_for_sync(self): - """ - Wait for sync to finish. - """ - wait = 0 - syncer = self.syncer - if syncer is not None: - while syncer.syncing: - time.sleep(WAIT_STEP) - wait += WAIT_STEP - if wait >= MAX_WAIT: - raise SyncTimeoutError - - def test_db_sync(self): - """ - Test sync. - - Adapted to check for encrypted content. - """ - doc1 = self.db1.create_doc_from_json(tests.simple_doc) - doc2 = self.db2.create_doc_from_json(tests.nested_doc) - d = self.do_sync('test') - - def _assert_successful_sync(results): - import time - # need to give time to the encryption to proceed - # TODO should implement a defer list to subscribe to the - # all-decrypted event - time.sleep(2) - local_gen_before_sync = results - self.wait_for_sync() - - gen, _, changes = self.db1.whats_changed(local_gen_before_sync) - self.assertEqual(1, len(changes)) - - self.assertEqual(doc2.doc_id, changes[0][0]) - self.assertEqual(1, gen - local_gen_before_sync) - - self.assertGetEncryptedDoc( - self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) - self.assertGetEncryptedDoc( - self.db1, doc2.doc_id, doc2.rev, tests.nested_doc, False) - - d.addCallback(_assert_successful_sync) - return d diff --git a/common/src/leap/soledad/common/tests/u1db_tests/README b/common/src/leap/soledad/common/tests/u1db_tests/README deleted file mode 100644 index 0525cfdb..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/README +++ /dev/null @@ -1,25 +0,0 @@ -General info ------------- - -Test files in this directory are derived from u1db-0.1.4 tests. The main -difference is that: - - (1) they include the test infrastructure packed with soledad; and - (2) they do not include c_backend_wrapper testing. - -Dependencies ------------- - -u1db tests depend on the following python packages: - - unittest2 - mercurial - hgtools - testtools - discover - oauth - testscenarios - dirspec - paste - routes - cython diff --git a/common/src/leap/soledad/common/tests/u1db_tests/__init__.py b/common/src/leap/soledad/common/tests/u1db_tests/__init__.py deleted file mode 100644 index 7f334b4a..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/__init__.py +++ /dev/null @@ -1,461 +0,0 @@ -# Copyright 2011-2012 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see . -""" -Test infrastructure for U1DB -""" - -import copy -import shutil -import socket -import tempfile -import threading -import json - -from wsgiref import simple_server - -from oauth import oauth -from pysqlcipher import dbapi2 -from StringIO import StringIO - -import testscenarios -from twisted.trial import unittest -from twisted.web.server import Site -from twisted.web.wsgi import WSGIResource -from twisted.internet import reactor - -from leap.soledad.common.l2db import errors -from leap.soledad.common.l2db import Document -from leap.soledad.common.l2db.backends import inmemory -from leap.soledad.common.l2db.backends import sqlite_backend -from leap.soledad.common.l2db.remote import server_state -from leap.soledad.common.l2db.remote import http_app -from leap.soledad.common.l2db.remote import http_target - - -class TestCase(unittest.TestCase): - - def createTempDir(self, prefix='u1db-tmp-'): - """Create a temporary directory to do some work in. - - This directory will be scheduled for cleanup when the test ends. - """ - tempdir = tempfile.mkdtemp(prefix=prefix) - self.addCleanup(shutil.rmtree, tempdir) - return tempdir - - def make_document(self, doc_id, doc_rev, content, has_conflicts=False): - return self.make_document_for_test( - self, doc_id, doc_rev, content, has_conflicts) - - def make_document_for_test(self, test, doc_id, doc_rev, content, - has_conflicts): - return make_document_for_test( - test, doc_id, doc_rev, content, has_conflicts) - - def assertGetDoc(self, db, doc_id, doc_rev, content, has_conflicts): - """Assert that the document in the database looks correct.""" - exp_doc = self.make_document(doc_id, doc_rev, content, - has_conflicts=has_conflicts) - self.assertEqual(exp_doc, db.get_doc(doc_id)) - - def assertGetDocIncludeDeleted(self, db, doc_id, doc_rev, content, - has_conflicts): - """Assert that the document in the database looks correct.""" - exp_doc = self.make_document(doc_id, doc_rev, content, - has_conflicts=has_conflicts) - self.assertEqual(exp_doc, db.get_doc(doc_id, include_deleted=True)) - - def assertGetDocConflicts(self, db, doc_id, conflicts): - """Assert what conflicts are stored for a given doc_id. - - :param conflicts: A list of (doc_rev, content) pairs. - The first item must match the first item returned from the - database, however the rest can be returned in any order. - """ - if conflicts: - conflicts = [(rev, - (json.loads(cont) if isinstance(cont, basestring) - else cont)) for (rev, cont) in conflicts] - conflicts = conflicts[:1] + sorted(conflicts[1:]) - actual = db.get_doc_conflicts(doc_id) - if actual: - actual = [ - (doc.rev, (json.loads(doc.get_json()) - if doc.get_json() is not None else None)) - for doc in actual] - actual = actual[:1] + sorted(actual[1:]) - self.assertEqual(conflicts, actual) - - -def multiply_scenarios(a_scenarios, b_scenarios): - """Create the cross-product of scenarios.""" - - all_scenarios = [] - for a_name, a_attrs in a_scenarios: - for b_name, b_attrs in b_scenarios: - name = '%s,%s' % (a_name, b_name) - attrs = dict(a_attrs) - attrs.update(b_attrs) - all_scenarios.append((name, attrs)) - return all_scenarios - - -simple_doc = '{"key": "value"}' -nested_doc = '{"key": "value", "sub": {"doc": "underneath"}}' - - -def make_memory_database_for_test(test, replica_uid): - return inmemory.InMemoryDatabase(replica_uid) - - -def copy_memory_database_for_test(test, db): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS - # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE - # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN - # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR - # HOUSE. - new_db = inmemory.InMemoryDatabase(db._replica_uid) - new_db._transaction_log = db._transaction_log[:] - new_db._docs = copy.deepcopy(db._docs) - new_db._conflicts = copy.deepcopy(db._conflicts) - new_db._indexes = copy.deepcopy(db._indexes) - new_db._factory = db._factory - return new_db - - -def make_sqlite_partial_expanded_for_test(test, replica_uid): - db = sqlite_backend.SQLitePartialExpandDatabase(':memory:') - db._set_replica_uid(replica_uid) - return db - - -def copy_sqlite_partial_expanded_for_test(test, db): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS - # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE - # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN - # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR - # HOUSE. - new_db = sqlite_backend.SQLitePartialExpandDatabase(':memory:') - tmpfile = StringIO() - for line in db._db_handle.iterdump(): - if 'sqlite_sequence' not in line: # work around bug in iterdump - tmpfile.write('%s\n' % line) - tmpfile.seek(0) - new_db._db_handle = dbapi2.connect(':memory:') - new_db._db_handle.cursor().executescript(tmpfile.read()) - new_db._db_handle.commit() - new_db._set_replica_uid(db._replica_uid) - new_db._factory = db._factory - return new_db - - -def make_document_for_test(test, doc_id, rev, content, has_conflicts=False): - return Document(doc_id, rev, content, has_conflicts=has_conflicts) - - -LOCAL_DATABASES_SCENARIOS = [ - ('mem', {'make_database_for_test': make_memory_database_for_test, - 'copy_database_for_test': copy_memory_database_for_test, - 'make_document_for_test': make_document_for_test}), - ('sql', {'make_database_for_test': - make_sqlite_partial_expanded_for_test, - 'copy_database_for_test': - copy_sqlite_partial_expanded_for_test, - 'make_document_for_test': make_document_for_test}), -] - - -class DatabaseBaseTests(TestCase): - - # set to True assertTransactionLog - # is happy with all trans ids = '' - accept_fixed_trans_id = False - - scenarios = LOCAL_DATABASES_SCENARIOS - - def make_database_for_test(self, replica_uid): - return make_memory_database_for_test(self, replica_uid) - - def create_database(self, *args): - return self.make_database_for_test(self, *args) - - def copy_database(self, db): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES - # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST - # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS - # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND - # NINJA TO YOUR HOUSE. - return self.copy_database_for_test(self, db) - - def setUp(self): - super(DatabaseBaseTests, self).setUp() - self.db = self.create_database('test') - - def tearDown(self): - if hasattr(self, 'db') and self.db is not None: - self.db.close() - super(DatabaseBaseTests, self).tearDown() - - def assertTransactionLog(self, doc_ids, db): - """Assert that the given docs are in the transaction log.""" - log = db._get_transaction_log() - just_ids = [] - seen_transactions = set() - for doc_id, transaction_id in log: - just_ids.append(doc_id) - self.assertIsNot(None, transaction_id, - "Transaction id should not be None") - if transaction_id == '' and self.accept_fixed_trans_id: - continue - self.assertNotEqual('', transaction_id, - "Transaction id should be a unique string") - self.assertTrue(transaction_id.startswith('T-')) - self.assertNotIn(transaction_id, seen_transactions) - seen_transactions.add(transaction_id) - self.assertEqual(doc_ids, just_ids) - - def getLastTransId(self, db): - """Return the transaction id for the last database update.""" - return self.db._get_transaction_log()[-1][-1] - - -class ServerStateForTests(server_state.ServerState): - - """Used in the test suite, so we don't have to touch disk, etc.""" - - def __init__(self): - super(ServerStateForTests, self).__init__() - self._dbs = {} - - def open_database(self, path): - try: - return self._dbs[path] - except KeyError: - raise errors.DatabaseDoesNotExist - - def check_database(self, path): - # cares only about the possible exception - self.open_database(path) - - def ensure_database(self, path): - try: - db = self.open_database(path) - except errors.DatabaseDoesNotExist: - db = self._create_database(path) - return db, db._replica_uid - - def _copy_database(self, db): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES - # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST - # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS - # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND - # NINJA TO YOUR HOUSE. - new_db = copy_memory_database_for_test(None, db) - path = db._replica_uid - while path in self._dbs: - path += 'copy' - self._dbs[path] = new_db - return new_db - - def _create_database(self, path): - db = inmemory.InMemoryDatabase(path) - self._dbs[path] = db - return db - - def delete_database(self, path): - del self._dbs[path] - - -class ResponderForTests(object): - - """Responder for tests.""" - _started = False - sent_response = False - status = None - - def start_response(self, status='success', **kwargs): - self._started = True - self.status = status - self.kwargs = kwargs - - def send_response(self, status='success', **kwargs): - self.start_response(status, **kwargs) - self.finish_response() - - def finish_response(self): - self.sent_response = True - - -class TestCaseWithServer(TestCase): - - @staticmethod - def server_def(): - # hook point - # should return (ServerClass, "shutdown method name", "url_scheme") - class _RequestHandler(simple_server.WSGIRequestHandler): - - def log_request(*args): - pass # suppress - - def make_server(host_port, application): - assert application, "forgot to override make_app(_with_state)?" - srv = simple_server.WSGIServer(host_port, _RequestHandler) - # patch the value in if it's None - if getattr(application, 'base_url', 1) is None: - application.base_url = "http://%s:%s" % srv.server_address - srv.set_app(application) - return srv - - return make_server, "shutdown", "http" - - @staticmethod - def make_app_with_state(state): - # hook point - return None - - def make_app(self): - # potential hook point - self.request_state = ServerStateForTests() - return self.make_app_with_state(self.request_state) - - def setUp(self): - super(TestCaseWithServer, self).setUp() - self.server = self.server_thread = self.port = None - - def tearDown(self): - if self.server is not None: - self.server.shutdown() - self.server_thread.join() - self.server.server_close() - if self.port: - self.port.stopListening() - super(TestCaseWithServer, self).tearDown() - - @property - def url_scheme(self): - return 'http' - - def startTwistedServer(self): - application = self.make_app() - resource = WSGIResource(reactor, reactor.getThreadPool(), application) - site = Site(resource) - self.port = reactor.listenTCP(0, site, interface='127.0.0.1') - host = self.port.getHost() - self.server_address = (host.host, host.port) - self.addCleanup(self.port.stopListening) - - def startServer(self): - server_def = self.server_def() - server_class, shutdown_meth, _ = server_def - application = self.make_app() - self.server = server_class(('127.0.0.1', 0), application) - self.server_thread = threading.Thread(target=self.server.serve_forever, - kwargs=dict(poll_interval=0.01)) - self.server_thread.start() - self.addCleanup(self.server_thread.join) - self.addCleanup(getattr(self.server, shutdown_meth)) - self.server_address = self.server.server_address - - def getURL(self, path=None): - host, port = self.server_address - if path is None: - path = '' - return '%s://%s:%s/%s' % (self.url_scheme, host, port, path) - - -def socket_pair(): - """Return a pair of TCP sockets connected to each other. - - Unlike socket.socketpair, this should work on Windows. - """ - sock_pair = getattr(socket, 'socket_pair', None) - if sock_pair: - return sock_pair(socket.AF_INET, socket.SOCK_STREAM) - listen_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - listen_sock.bind(('127.0.0.1', 0)) - listen_sock.listen(1) - client_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - client_sock.connect(listen_sock.getsockname()) - server_sock, addr = listen_sock.accept() - listen_sock.close() - return server_sock, client_sock - - -# OAuth related testing - -consumer1 = oauth.OAuthConsumer('K1', 'S1') -token1 = oauth.OAuthToken('kkkk1', 'XYZ') -consumer2 = oauth.OAuthConsumer('K2', 'S2') -token2 = oauth.OAuthToken('kkkk2', 'ZYX') -token3 = oauth.OAuthToken('kkkk3', 'ZYX') - - -class TestingOAuthDataStore(oauth.OAuthDataStore): - - """In memory predefined OAuthDataStore for testing.""" - - consumers = { - consumer1.key: consumer1, - consumer2.key: consumer2, - } - - tokens = { - token1.key: token1, - token2.key: token2 - } - - def lookup_consumer(self, key): - return self.consumers.get(key) - - def lookup_token(self, token_type, token_token): - return self.tokens.get(token_token) - - def lookup_nonce(self, oauth_consumer, oauth_token, nonce): - return None - -testingOAuthStore = TestingOAuthDataStore() - -sign_meth_HMAC_SHA1 = oauth.OAuthSignatureMethod_HMAC_SHA1() -sign_meth_PLAINTEXT = oauth.OAuthSignatureMethod_PLAINTEXT() - - -def load_with_scenarios(loader, standard_tests, pattern): - """Load the tests in a given module. - - This just applies testscenarios.generate_scenarios to all the tests that - are present. We do it at load time rather than at run time, because it - plays nicer with various tools. - """ - suite = loader.suiteClass() - suite.addTests(testscenarios.generate_scenarios(standard_tests)) - return suite - - -# from u1db.tests.test_remote_sync_target - -def make_http_app(state): - return http_app.HTTPApp(state) - - -def http_sync_target(test, path): - return http_target.HTTPSyncTarget(test.getURL(path)) - - -def make_oauth_http_app(state): - app = http_app.HTTPApp(state) - application = oauth_middleware.OAuthMiddleware(app, None, prefix='/~/') - application.get_oauth_data_store = lambda: tests.testingOAuthStore - return application diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py b/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py deleted file mode 100644 index c0c6ea6b..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_backends.py +++ /dev/null @@ -1,1915 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# Copyright 2016 LEAP Encryption Access Project -# -# This file is part of leap.soledad.common -# -# leap.soledad.common is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see . - -""" -The backend class for L2DB. This deals with hiding storage details. -""" - -import json - -from leap.soledad.common.l2db import DocumentBase -from leap.soledad.common.l2db import errors -from leap.soledad.common.l2db import vectorclock -from leap.soledad.common.l2db.remote import http_database - -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests.u1db_tests import make_http_app -from leap.soledad.common.tests.u1db_tests import make_oauth_http_app - -from unittest import skip - -simple_doc = tests.simple_doc -nested_doc = tests.nested_doc - - -def make_http_database_for_test(test, replica_uid, path='test', *args): - test.startServer() - test.request_state._create_database(replica_uid) - return http_database.HTTPDatabase(test.getURL(path)) - - -def copy_http_database_for_test(test, db): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS - # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE - # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN - # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR - # HOUSE. - return test.request_state._copy_database(db) - - -def make_oauth_http_database_for_test(test, replica_uid): - http_db = make_http_database_for_test(test, replica_uid, '~/test') - http_db.set_oauth_credentials(tests.consumer1.key, tests.consumer1.secret, - tests.token1.key, tests.token1.secret) - return http_db - - -def copy_oauth_http_database_for_test(test, db): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS - # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE - # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN - # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR - # HOUSE. - http_db = test.request_state._copy_database(db) - http_db.set_oauth_credentials(tests.consumer1.key, tests.consumer1.secret, - tests.token1.key, tests.token1.secret) - return http_db - - -class TestAlternativeDocument(DocumentBase): - - """A (not very) alternative implementation of Document.""" - - -@skip("Skiping tests imported from U1DB.") -class AllDatabaseTests(tests.DatabaseBaseTests, tests.TestCaseWithServer): - - scenarios = tests.LOCAL_DATABASES_SCENARIOS + [ - ('http', {'make_database_for_test': make_http_database_for_test, - 'copy_database_for_test': copy_http_database_for_test, - 'make_document_for_test': tests.make_document_for_test, - 'make_app_with_state': make_http_app}), - ('oauth_http', {'make_database_for_test': - make_oauth_http_database_for_test, - 'copy_database_for_test': - copy_oauth_http_database_for_test, - 'make_document_for_test': tests.make_document_for_test, - 'make_app_with_state': make_oauth_http_app}) - ] - - def test_close(self): - self.db.close() - - def test_create_doc_allocating_doc_id(self): - doc = self.db.create_doc_from_json(simple_doc) - self.assertNotEqual(None, doc.doc_id) - self.assertNotEqual(None, doc.rev) - self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) - - def test_create_doc_different_ids_same_db(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(nested_doc) - self.assertNotEqual(doc1.doc_id, doc2.doc_id) - - def test_create_doc_with_id(self): - doc = self.db.create_doc_from_json(simple_doc, doc_id='my-id') - self.assertEqual('my-id', doc.doc_id) - self.assertNotEqual(None, doc.rev) - self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) - - def test_create_doc_existing_id(self): - doc = self.db.create_doc_from_json(simple_doc) - new_content = '{"something": "else"}' - self.assertRaises( - errors.RevisionConflict, self.db.create_doc_from_json, - new_content, doc.doc_id) - self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) - - def test_put_doc_creating_initial(self): - doc = self.make_document('my_doc_id', None, simple_doc) - new_rev = self.db.put_doc(doc) - self.assertIsNot(None, new_rev) - self.assertGetDoc(self.db, 'my_doc_id', new_rev, simple_doc, False) - - def test_put_doc_space_in_id(self): - doc = self.make_document('my doc id', None, simple_doc) - self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) - - def test_put_doc_update(self): - doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') - orig_rev = doc.rev - doc.set_json('{"updated": "stuff"}') - new_rev = self.db.put_doc(doc) - self.assertNotEqual(new_rev, orig_rev) - self.assertGetDoc(self.db, 'my_doc_id', new_rev, - '{"updated": "stuff"}', False) - self.assertEqual(doc.rev, new_rev) - - def test_put_non_ascii_key(self): - content = json.dumps({u'key\xe5': u'val'}) - doc = self.db.create_doc_from_json(content, doc_id='my_doc') - self.assertGetDoc(self.db, 'my_doc', doc.rev, content, False) - - def test_put_non_ascii_value(self): - content = json.dumps({'key': u'\xe5'}) - doc = self.db.create_doc_from_json(content, doc_id='my_doc') - self.assertGetDoc(self.db, 'my_doc', doc.rev, content, False) - - def test_put_doc_refuses_no_id(self): - doc = self.make_document(None, None, simple_doc) - self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) - doc = self.make_document("", None, simple_doc) - self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) - - def test_put_doc_refuses_slashes(self): - doc = self.make_document('a/b', None, simple_doc) - self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) - doc = self.make_document(r'\b', None, simple_doc) - self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) - - def test_put_doc_url_quoting_is_fine(self): - doc_id = "%2F%2Ffoo%2Fbar" - doc = self.make_document(doc_id, None, simple_doc) - new_rev = self.db.put_doc(doc) - self.assertGetDoc(self.db, doc_id, new_rev, simple_doc, False) - - def test_put_doc_refuses_non_existing_old_rev(self): - doc = self.make_document('doc-id', 'test:4', simple_doc) - self.assertRaises(errors.RevisionConflict, self.db.put_doc, doc) - - def test_put_doc_refuses_non_ascii_doc_id(self): - doc = self.make_document('d\xc3\xa5c-id', None, simple_doc) - self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) - - def test_put_fails_with_bad_old_rev(self): - doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') - old_rev = doc.rev - bad_doc = self.make_document(doc.doc_id, 'other:1', - '{"something": "else"}') - self.assertRaises(errors.RevisionConflict, self.db.put_doc, bad_doc) - self.assertGetDoc(self.db, 'my_doc_id', old_rev, simple_doc, False) - - def test_create_succeeds_after_delete(self): - doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') - self.db.delete_doc(doc) - deleted_doc = self.db.get_doc('my_doc_id', include_deleted=True) - deleted_vc = vectorclock.VectorClockRev(deleted_doc.rev) - new_doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') - self.assertGetDoc(self.db, 'my_doc_id', new_doc.rev, simple_doc, False) - new_vc = vectorclock.VectorClockRev(new_doc.rev) - self.assertTrue( - new_vc.is_newer(deleted_vc), - "%s does not supersede %s" % (new_doc.rev, deleted_doc.rev)) - - def test_put_succeeds_after_delete(self): - doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') - self.db.delete_doc(doc) - deleted_doc = self.db.get_doc('my_doc_id', include_deleted=True) - deleted_vc = vectorclock.VectorClockRev(deleted_doc.rev) - doc2 = self.make_document('my_doc_id', None, simple_doc) - self.db.put_doc(doc2) - self.assertGetDoc(self.db, 'my_doc_id', doc2.rev, simple_doc, False) - new_vc = vectorclock.VectorClockRev(doc2.rev) - self.assertTrue( - new_vc.is_newer(deleted_vc), - "%s does not supersede %s" % (doc2.rev, deleted_doc.rev)) - - def test_get_doc_after_put(self): - doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') - self.assertGetDoc(self.db, 'my_doc_id', doc.rev, simple_doc, False) - - def test_get_doc_nonexisting(self): - self.assertIs(None, self.db.get_doc('non-existing')) - - def test_get_doc_deleted(self): - doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') - self.db.delete_doc(doc) - self.assertIs(None, self.db.get_doc('my_doc_id')) - - def test_get_doc_include_deleted(self): - doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') - self.db.delete_doc(doc) - self.assertGetDocIncludeDeleted( - self.db, doc.doc_id, doc.rev, None, False) - - def test_get_docs(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(nested_doc) - self.assertEqual([doc1, doc2], - list(self.db.get_docs([doc1.doc_id, doc2.doc_id]))) - - def test_get_docs_deleted(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(nested_doc) - self.db.delete_doc(doc1) - self.assertEqual([doc2], - list(self.db.get_docs([doc1.doc_id, doc2.doc_id]))) - - def test_get_docs_include_deleted(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(nested_doc) - self.db.delete_doc(doc1) - self.assertEqual( - [doc1, doc2], - list(self.db.get_docs([doc1.doc_id, doc2.doc_id], - include_deleted=True))) - - def test_get_docs_request_ordered(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(nested_doc) - self.assertEqual([doc1, doc2], - list(self.db.get_docs([doc1.doc_id, doc2.doc_id]))) - self.assertEqual([doc2, doc1], - list(self.db.get_docs([doc2.doc_id, doc1.doc_id]))) - - def test_get_docs_empty_list(self): - self.assertEqual([], list(self.db.get_docs([]))) - - def test_handles_nested_content(self): - doc = self.db.create_doc_from_json(nested_doc) - self.assertGetDoc(self.db, doc.doc_id, doc.rev, nested_doc, False) - - def test_handles_doc_with_null(self): - doc = self.db.create_doc_from_json('{"key": null}') - self.assertGetDoc(self.db, doc.doc_id, doc.rev, '{"key": null}', False) - - def test_delete_doc(self): - doc = self.db.create_doc_from_json(simple_doc) - self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) - orig_rev = doc.rev - self.db.delete_doc(doc) - self.assertNotEqual(orig_rev, doc.rev) - self.assertGetDocIncludeDeleted( - self.db, doc.doc_id, doc.rev, None, False) - self.assertIs(None, self.db.get_doc(doc.doc_id)) - - def test_delete_doc_non_existent(self): - doc = self.make_document('non-existing', 'other:1', simple_doc) - self.assertRaises(errors.DocumentDoesNotExist, self.db.delete_doc, doc) - - def test_delete_doc_already_deleted(self): - doc = self.db.create_doc_from_json(simple_doc) - self.db.delete_doc(doc) - self.assertRaises(errors.DocumentAlreadyDeleted, - self.db.delete_doc, doc) - self.assertGetDocIncludeDeleted( - self.db, doc.doc_id, doc.rev, None, False) - - def test_delete_doc_bad_rev(self): - doc1 = self.db.create_doc_from_json(simple_doc) - self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) - doc2 = self.make_document(doc1.doc_id, 'other:1', simple_doc) - self.assertRaises(errors.RevisionConflict, self.db.delete_doc, doc2) - self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) - - def test_delete_doc_sets_content_to_None(self): - doc = self.db.create_doc_from_json(simple_doc) - self.db.delete_doc(doc) - self.assertIs(None, doc.get_json()) - - def test_delete_doc_rev_supersedes(self): - doc = self.db.create_doc_from_json(simple_doc) - doc.set_json(nested_doc) - self.db.put_doc(doc) - doc.set_json('{"fishy": "content"}') - self.db.put_doc(doc) - old_rev = doc.rev - self.db.delete_doc(doc) - cur_vc = vectorclock.VectorClockRev(old_rev) - deleted_vc = vectorclock.VectorClockRev(doc.rev) - self.assertTrue(deleted_vc.is_newer(cur_vc), - "%s does not supersede %s" % (doc.rev, old_rev)) - - def test_delete_then_put(self): - doc = self.db.create_doc_from_json(simple_doc) - self.db.delete_doc(doc) - self.assertGetDocIncludeDeleted( - self.db, doc.doc_id, doc.rev, None, False) - doc.set_json(nested_doc) - self.db.put_doc(doc) - self.assertGetDoc(self.db, doc.doc_id, doc.rev, nested_doc, False) - - -@skip("Skiping tests imported from U1DB.") -class DocumentSizeTests(tests.DatabaseBaseTests): - - scenarios = tests.LOCAL_DATABASES_SCENARIOS - - def test_put_doc_refuses_oversized_documents(self): - self.db.set_document_size_limit(1) - doc = self.make_document('doc-id', None, simple_doc) - self.assertRaises(errors.DocumentTooBig, self.db.put_doc, doc) - - def test_create_doc_refuses_oversized_documents(self): - self.db.set_document_size_limit(1) - self.assertRaises( - errors.DocumentTooBig, self.db.create_doc_from_json, simple_doc, - doc_id='my_doc_id') - - def test_set_document_size_limit_zero(self): - self.db.set_document_size_limit(0) - self.assertEqual(0, self.db.document_size_limit) - - def test_set_document_size_limit(self): - self.db.set_document_size_limit(1000000) - self.assertEqual(1000000, self.db.document_size_limit) - - -@skip("Skiping tests imported from U1DB.") -class LocalDatabaseTests(tests.DatabaseBaseTests): - - scenarios = tests.LOCAL_DATABASES_SCENARIOS - - def setUp(self): - tests.DatabaseBaseTests.setUp(self) - - def test_create_doc_different_ids_diff_db(self): - doc1 = self.db.create_doc_from_json(simple_doc) - db2 = self.create_database('other-uid') - doc2 = db2.create_doc_from_json(simple_doc) - self.assertNotEqual(doc1.doc_id, doc2.doc_id) - db2.close() - - def test_put_doc_refuses_slashes_picky(self): - doc = self.make_document('/a', None, simple_doc) - self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) - - def test_get_all_docs_empty(self): - self.assertEqual([], list(self.db.get_all_docs()[1])) - - def test_get_all_docs(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(nested_doc) - self.assertEqual( - sorted([doc1, doc2]), sorted(list(self.db.get_all_docs()[1]))) - - def test_get_all_docs_exclude_deleted(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(nested_doc) - self.db.delete_doc(doc2) - self.assertEqual([doc1], list(self.db.get_all_docs()[1])) - - def test_get_all_docs_include_deleted(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(nested_doc) - self.db.delete_doc(doc2) - self.assertEqual( - sorted([doc1, doc2]), - sorted(list(self.db.get_all_docs(include_deleted=True)[1]))) - - def test_get_all_docs_generation(self): - self.db.create_doc_from_json(simple_doc) - self.db.create_doc_from_json(nested_doc) - self.assertEqual(2, self.db.get_all_docs()[0]) - - def test_simple_put_doc_if_newer(self): - doc = self.make_document('my-doc-id', 'test:1', simple_doc) - state_at_gen = self.db._put_doc_if_newer( - doc, save_conflict=False, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual(('inserted', 1), state_at_gen) - self.assertGetDoc(self.db, 'my-doc-id', 'test:1', simple_doc, False) - - def test_simple_put_doc_if_newer_deleted(self): - self.db.create_doc_from_json('{}', doc_id='my-doc-id') - doc = self.make_document('my-doc-id', 'test:2', None) - state_at_gen = self.db._put_doc_if_newer( - doc, save_conflict=False, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual(('inserted', 2), state_at_gen) - self.assertGetDocIncludeDeleted( - self.db, 'my-doc-id', 'test:2', None, False) - - def test_put_doc_if_newer_already_superseded(self): - orig_doc = '{"new": "doc"}' - doc1 = self.db.create_doc_from_json(orig_doc) - doc1_rev1 = doc1.rev - doc1.set_json(simple_doc) - self.db.put_doc(doc1) - doc1_rev2 = doc1.rev - # Nothing is inserted, because the document is already superseded - doc = self.make_document(doc1.doc_id, doc1_rev1, orig_doc) - state, _ = self.db._put_doc_if_newer( - doc, save_conflict=False, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual('superseded', state) - self.assertGetDoc(self.db, doc1.doc_id, doc1_rev2, simple_doc, False) - - def test_put_doc_if_newer_autoresolve(self): - doc1 = self.db.create_doc_from_json(simple_doc) - rev = doc1.rev - doc = self.make_document(doc1.doc_id, "whatever:1", doc1.get_json()) - state, _ = self.db._put_doc_if_newer( - doc, save_conflict=False, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual('superseded', state) - doc2 = self.db.get_doc(doc1.doc_id) - v2 = vectorclock.VectorClockRev(doc2.rev) - self.assertTrue(v2.is_newer(vectorclock.VectorClockRev("whatever:1"))) - self.assertTrue(v2.is_newer(vectorclock.VectorClockRev(rev))) - # strictly newer locally - self.assertTrue(rev not in doc2.rev) - - def test_put_doc_if_newer_already_converged(self): - orig_doc = '{"new": "doc"}' - doc1 = self.db.create_doc_from_json(orig_doc) - state_at_gen = self.db._put_doc_if_newer( - doc1, save_conflict=False, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual(('converged', 1), state_at_gen) - - def test_put_doc_if_newer_conflicted(self): - doc1 = self.db.create_doc_from_json(simple_doc) - # Nothing is inserted, the document id is returned as would-conflict - alt_doc = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - state, _ = self.db._put_doc_if_newer( - alt_doc, save_conflict=False, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual('conflicted', state) - # The database wasn't altered - self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) - - def test_put_doc_if_newer_newer_generation(self): - self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') - doc = self.make_document('doc_id', 'other:2', simple_doc) - state, _ = self.db._put_doc_if_newer( - doc, save_conflict=False, replica_uid='other', replica_gen=2, - replica_trans_id='T-irrelevant') - self.assertEqual('inserted', state) - - def test_put_doc_if_newer_same_generation_same_txid(self): - self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') - doc = self.db.create_doc_from_json(simple_doc) - self.make_document(doc.doc_id, 'other:1', simple_doc) - state, _ = self.db._put_doc_if_newer( - doc, save_conflict=False, replica_uid='other', replica_gen=1, - replica_trans_id='T-sid') - self.assertEqual('converged', state) - - def test_put_doc_if_newer_wrong_transaction_id(self): - self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') - doc = self.make_document('doc_id', 'other:1', simple_doc) - self.assertRaises( - errors.InvalidTransactionId, - self.db._put_doc_if_newer, doc, save_conflict=False, - replica_uid='other', replica_gen=1, replica_trans_id='T-sad') - - def test_put_doc_if_newer_old_generation_older_doc(self): - orig_doc = '{"new": "doc"}' - doc = self.db.create_doc_from_json(orig_doc) - doc_rev1 = doc.rev - doc.set_json(simple_doc) - self.db.put_doc(doc) - self.db._set_replica_gen_and_trans_id('other', 3, 'T-sid') - older_doc = self.make_document(doc.doc_id, doc_rev1, simple_doc) - state, _ = self.db._put_doc_if_newer( - older_doc, save_conflict=False, replica_uid='other', replica_gen=8, - replica_trans_id='T-irrelevant') - self.assertEqual('superseded', state) - - def test_put_doc_if_newer_old_generation_newer_doc(self): - self.db._set_replica_gen_and_trans_id('other', 5, 'T-sid') - doc = self.make_document('doc_id', 'other:1', simple_doc) - self.assertRaises( - errors.InvalidGeneration, - self.db._put_doc_if_newer, doc, save_conflict=False, - replica_uid='other', replica_gen=1, replica_trans_id='T-sad') - - def test_put_doc_if_newer_replica_uid(self): - doc1 = self.db.create_doc_from_json(simple_doc) - self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') - doc2 = self.make_document(doc1.doc_id, doc1.rev + '|other:1', - nested_doc) - self.assertEqual('inserted', - self.db._put_doc_if_newer( - doc2, - save_conflict=False, - replica_uid='other', - replica_gen=2, - replica_trans_id='T-id2')[0]) - self.assertEqual((2, 'T-id2'), self.db._get_replica_gen_and_trans_id( - 'other')) - # Compare to the old rev, should be superseded - doc2 = self.make_document(doc1.doc_id, doc1.rev, nested_doc) - self.assertEqual('superseded', - self.db._put_doc_if_newer( - doc2, - save_conflict=False, - replica_uid='other', - replica_gen=3, - replica_trans_id='T-id3')[0]) - self.assertEqual( - (3, 'T-id3'), self.db._get_replica_gen_and_trans_id('other')) - # A conflict that isn't saved still records the sync gen, because we - # don't need to see it again - doc2 = self.make_document(doc1.doc_id, doc1.rev + '|fourth:1', - '{}') - self.assertEqual('conflicted', - self.db._put_doc_if_newer( - doc2, - save_conflict=False, - replica_uid='other', - replica_gen=4, - replica_trans_id='T-id4')[0]) - self.assertEqual( - (4, 'T-id4'), self.db._get_replica_gen_and_trans_id('other')) - - def test__get_replica_gen_and_trans_id(self): - self.assertEqual( - (0, ''), self.db._get_replica_gen_and_trans_id('other-db')) - self.db._set_replica_gen_and_trans_id('other-db', 2, 'T-transaction') - self.assertEqual( - (2, 'T-transaction'), - self.db._get_replica_gen_and_trans_id('other-db')) - - def test_put_updates_transaction_log(self): - doc = self.db.create_doc_from_json(simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - doc.set_json('{"something": "else"}') - self.db.put_doc(doc) - self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual((2, last_trans_id, [(doc.doc_id, 2, last_trans_id)]), - self.db.whats_changed()) - - def test_delete_updates_transaction_log(self): - doc = self.db.create_doc_from_json(simple_doc) - db_gen, _, _ = self.db.whats_changed() - self.db.delete_doc(doc) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual((2, last_trans_id, [(doc.doc_id, 2, last_trans_id)]), - self.db.whats_changed(db_gen)) - - def test_whats_changed_initial_database(self): - self.assertEqual((0, '', []), self.db.whats_changed()) - - def test_whats_changed_returns_one_id_for_multiple_changes(self): - doc = self.db.create_doc_from_json(simple_doc) - doc.set_json('{"new": "contents"}') - self.db.put_doc(doc) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual((2, last_trans_id, [(doc.doc_id, 2, last_trans_id)]), - self.db.whats_changed()) - self.assertEqual((2, last_trans_id, []), self.db.whats_changed(2)) - - def test_whats_changed_returns_last_edits_ascending(self): - doc = self.db.create_doc_from_json(simple_doc) - doc1 = self.db.create_doc_from_json(simple_doc) - doc.set_json('{"new": "contents"}') - self.db.delete_doc(doc1) - delete_trans_id = self.getLastTransId(self.db) - self.db.put_doc(doc) - put_trans_id = self.getLastTransId(self.db) - self.assertEqual((4, put_trans_id, - [(doc1.doc_id, 3, delete_trans_id), - (doc.doc_id, 4, put_trans_id)]), - self.db.whats_changed()) - - def test_whats_changed_doesnt_include_old_gen(self): - self.db.create_doc_from_json(simple_doc) - self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(simple_doc) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual((3, last_trans_id, [(doc2.doc_id, 3, last_trans_id)]), - self.db.whats_changed(2)) - - -@skip("Skiping tests imported from U1DB.") -class LocalDatabaseValidateGenNTransIdTests(tests.DatabaseBaseTests): - - scenarios = tests.LOCAL_DATABASES_SCENARIOS - - def test_validate_gen_and_trans_id(self): - self.db.create_doc_from_json(simple_doc) - gen, trans_id = self.db._get_generation_info() - self.db.validate_gen_and_trans_id(gen, trans_id) - - def test_validate_gen_and_trans_id_invalid_txid(self): - self.db.create_doc_from_json(simple_doc) - gen, _ = self.db._get_generation_info() - self.assertRaises( - errors.InvalidTransactionId, - self.db.validate_gen_and_trans_id, gen, 'wrong') - - def test_validate_gen_and_trans_id_invalid_gen(self): - self.db.create_doc_from_json(simple_doc) - gen, trans_id = self.db._get_generation_info() - self.assertRaises( - errors.InvalidGeneration, - self.db.validate_gen_and_trans_id, gen + 1, trans_id) - - -@skip("Skiping tests imported from U1DB.") -class LocalDatabaseValidateSourceGenTests(tests.DatabaseBaseTests): - - scenarios = tests.LOCAL_DATABASES_SCENARIOS - - def test_validate_source_gen_and_trans_id_same(self): - self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') - self.db._validate_source('other', 1, 'T-sid') - - def test_validate_source_gen_newer(self): - self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') - self.db._validate_source('other', 2, 'T-whatevs') - - def test_validate_source_wrong_txid(self): - self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') - self.assertRaises( - errors.InvalidTransactionId, - self.db._validate_source, 'other', 1, 'T-sad') - - -@skip("Skiping tests imported from U1DB.") -class LocalDatabaseWithConflictsTests(tests.DatabaseBaseTests): - # test supporting/functionality around storing conflicts - - scenarios = tests.LOCAL_DATABASES_SCENARIOS - - def test_get_docs_conflicted(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual([doc2], list(self.db.get_docs([doc1.doc_id]))) - - def test_get_docs_conflicts_ignored(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(nested_doc) - alt_doc = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - no_conflict_doc = self.make_document(doc1.doc_id, 'alternate:1', - nested_doc) - self.assertEqual([no_conflict_doc, doc2], - list(self.db.get_docs([doc1.doc_id, doc2.doc_id], - check_for_conflicts=False))) - - def test_get_doc_conflicts(self): - doc = self.db.create_doc_from_json(simple_doc) - alt_doc = self.make_document(doc.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual([alt_doc, doc], - self.db.get_doc_conflicts(doc.doc_id)) - - def test_get_all_docs_sees_conflicts(self): - doc = self.db.create_doc_from_json(simple_doc) - alt_doc = self.make_document(doc.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - _, docs = self.db.get_all_docs() - self.assertTrue(list(docs)[0].has_conflicts) - - def test_get_doc_conflicts_unconflicted(self): - doc = self.db.create_doc_from_json(simple_doc) - self.assertEqual([], self.db.get_doc_conflicts(doc.doc_id)) - - def test_get_doc_conflicts_no_such_id(self): - self.assertEqual([], self.db.get_doc_conflicts('doc-id')) - - def test_resolve_doc(self): - doc = self.db.create_doc_from_json(simple_doc) - alt_doc = self.make_document(doc.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertGetDocConflicts(self.db, doc.doc_id, - [('alternate:1', nested_doc), - (doc.rev, simple_doc)]) - orig_rev = doc.rev - self.db.resolve_doc(doc, [alt_doc.rev, doc.rev]) - self.assertNotEqual(orig_rev, doc.rev) - self.assertFalse(doc.has_conflicts) - self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) - self.assertGetDocConflicts(self.db, doc.doc_id, []) - - def test_resolve_doc_picks_biggest_vcr(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertGetDocConflicts(self.db, doc1.doc_id, - [(doc2.rev, nested_doc), - (doc1.rev, simple_doc)]) - orig_doc1_rev = doc1.rev - self.db.resolve_doc(doc1, [doc2.rev, doc1.rev]) - self.assertFalse(doc1.has_conflicts) - self.assertNotEqual(orig_doc1_rev, doc1.rev) - self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) - self.assertGetDocConflicts(self.db, doc1.doc_id, []) - vcr_1 = vectorclock.VectorClockRev(orig_doc1_rev) - vcr_2 = vectorclock.VectorClockRev(doc2.rev) - vcr_new = vectorclock.VectorClockRev(doc1.rev) - self.assertTrue(vcr_new.is_newer(vcr_1)) - self.assertTrue(vcr_new.is_newer(vcr_2)) - - def test_resolve_doc_partial_not_winning(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertGetDocConflicts(self.db, doc1.doc_id, - [(doc2.rev, nested_doc), - (doc1.rev, simple_doc)]) - content3 = '{"key": "valin3"}' - doc3 = self.make_document(doc1.doc_id, 'third:1', content3) - self.db._put_doc_if_newer( - doc3, save_conflict=True, replica_uid='r', replica_gen=2, - replica_trans_id='bar') - self.assertGetDocConflicts(self.db, doc1.doc_id, - [(doc3.rev, content3), - (doc1.rev, simple_doc), - (doc2.rev, nested_doc)]) - self.db.resolve_doc(doc1, [doc2.rev, doc1.rev]) - self.assertTrue(doc1.has_conflicts) - self.assertGetDoc(self.db, doc1.doc_id, doc3.rev, content3, True) - self.assertGetDocConflicts(self.db, doc1.doc_id, - [(doc3.rev, content3), - (doc1.rev, simple_doc)]) - - def test_resolve_doc_partial_winning(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - content3 = '{"key": "valin3"}' - doc3 = self.make_document(doc1.doc_id, 'third:1', content3) - self.db._put_doc_if_newer( - doc3, save_conflict=True, replica_uid='r', replica_gen=2, - replica_trans_id='bar') - self.assertGetDocConflicts(self.db, doc1.doc_id, - [(doc3.rev, content3), - (doc1.rev, simple_doc), - (doc2.rev, nested_doc)]) - self.db.resolve_doc(doc1, [doc3.rev, doc1.rev]) - self.assertTrue(doc1.has_conflicts) - self.assertGetDocConflicts(self.db, doc1.doc_id, - [(doc1.rev, simple_doc), - (doc2.rev, nested_doc)]) - - def test_resolve_doc_with_delete_conflict(self): - doc1 = self.db.create_doc_from_json(simple_doc) - self.db.delete_doc(doc1) - doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertGetDocConflicts(self.db, doc1.doc_id, - [(doc2.rev, nested_doc), - (doc1.rev, None)]) - self.db.resolve_doc(doc2, [doc1.rev, doc2.rev]) - self.assertGetDocConflicts(self.db, doc1.doc_id, []) - self.assertGetDoc(self.db, doc2.doc_id, doc2.rev, nested_doc, False) - - def test_resolve_doc_with_delete_to_delete(self): - doc1 = self.db.create_doc_from_json(simple_doc) - self.db.delete_doc(doc1) - doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertGetDocConflicts(self.db, doc1.doc_id, - [(doc2.rev, nested_doc), - (doc1.rev, None)]) - self.db.resolve_doc(doc1, [doc1.rev, doc2.rev]) - self.assertGetDocConflicts(self.db, doc1.doc_id, []) - self.assertGetDocIncludeDeleted( - self.db, doc1.doc_id, doc1.rev, None, False) - - def test_put_doc_if_newer_save_conflicted(self): - doc1 = self.db.create_doc_from_json(simple_doc) - # Document is inserted as a conflict - doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - state, _ = self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual('conflicted', state) - # The database was updated - self.assertGetDoc(self.db, doc1.doc_id, doc2.rev, nested_doc, True) - - def test_force_doc_conflict_supersedes_properly(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.make_document(doc1.doc_id, 'alternate:1', '{"b": 1}') - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - doc3 = self.make_document(doc1.doc_id, 'altalt:1', '{"c": 1}') - self.db._put_doc_if_newer( - doc3, save_conflict=True, replica_uid='r', replica_gen=2, - replica_trans_id='bar') - doc22 = self.make_document(doc1.doc_id, 'alternate:2', '{"b": 2}') - self.db._put_doc_if_newer( - doc22, save_conflict=True, replica_uid='r', replica_gen=3, - replica_trans_id='zed') - self.assertGetDocConflicts(self.db, doc1.doc_id, - [('alternate:2', doc22.get_json()), - ('altalt:1', doc3.get_json()), - (doc1.rev, simple_doc)]) - - def test_put_doc_if_newer_save_conflict_was_deleted(self): - doc1 = self.db.create_doc_from_json(simple_doc) - self.db.delete_doc(doc1) - doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertTrue(doc2.has_conflicts) - self.assertGetDoc( - self.db, doc1.doc_id, 'alternate:1', nested_doc, True) - self.assertGetDocConflicts(self.db, doc1.doc_id, - [('alternate:1', nested_doc), - (doc1.rev, None)]) - - def test_put_doc_if_newer_propagates_full_resolution(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - resolved_vcr = vectorclock.VectorClockRev(doc1.rev) - vcr_2 = vectorclock.VectorClockRev(doc2.rev) - resolved_vcr.maximize(vcr_2) - resolved_vcr.increment('alternate') - doc_resolved = self.make_document(doc1.doc_id, resolved_vcr.as_str(), - '{"good": 1}') - state, _ = self.db._put_doc_if_newer( - doc_resolved, save_conflict=True, replica_uid='r', replica_gen=2, - replica_trans_id='foo2') - self.assertEqual('inserted', state) - self.assertFalse(doc_resolved.has_conflicts) - self.assertGetDocConflicts(self.db, doc1.doc_id, []) - doc3 = self.db.get_doc(doc1.doc_id) - self.assertFalse(doc3.has_conflicts) - - def test_put_doc_if_newer_propagates_partial_resolution(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.make_document(doc1.doc_id, 'altalt:1', '{}') - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - doc3 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - doc3, save_conflict=True, replica_uid='r', replica_gen=2, - replica_trans_id='foo2') - self.assertGetDocConflicts(self.db, doc1.doc_id, - [('alternate:1', nested_doc), - ('test:1', simple_doc), - ('altalt:1', '{}')]) - resolved_vcr = vectorclock.VectorClockRev(doc1.rev) - vcr_3 = vectorclock.VectorClockRev(doc3.rev) - resolved_vcr.maximize(vcr_3) - resolved_vcr.increment('alternate') - doc_resolved = self.make_document(doc1.doc_id, resolved_vcr.as_str(), - '{"good": 1}') - state, _ = self.db._put_doc_if_newer( - doc_resolved, save_conflict=True, replica_uid='r', replica_gen=3, - replica_trans_id='foo3') - self.assertEqual('inserted', state) - self.assertTrue(doc_resolved.has_conflicts) - doc4 = self.db.get_doc(doc1.doc_id) - self.assertTrue(doc4.has_conflicts) - self.assertGetDocConflicts(self.db, doc1.doc_id, - [('alternate:2|test:1', '{"good": 1}'), - ('altalt:1', '{}')]) - - def test_put_doc_if_newer_replica_uid(self): - doc1 = self.db.create_doc_from_json(simple_doc) - self.db._set_replica_gen_and_trans_id('other', 1, 'T-id') - doc2 = self.make_document(doc1.doc_id, doc1.rev + '|other:1', - nested_doc) - self.db._put_doc_if_newer(doc2, save_conflict=True, - replica_uid='other', replica_gen=2, - replica_trans_id='T-id2') - # Conflict vs the current update - doc2 = self.make_document(doc1.doc_id, doc1.rev + '|third:3', - '{}') - self.assertEqual('conflicted', - self.db._put_doc_if_newer( - doc2, - save_conflict=True, - replica_uid='other', - replica_gen=3, - replica_trans_id='T-id3')[0]) - self.assertEqual( - (3, 'T-id3'), self.db._get_replica_gen_and_trans_id('other')) - - def test_put_doc_if_newer_autoresolve_2(self): - # this is an ordering variant of _3, but that already works - # adding the test explicitly to catch the regression easily - doc_a1 = self.db.create_doc_from_json(simple_doc) - doc_a2 = self.make_document(doc_a1.doc_id, 'test:2', "{}") - doc_a1b1 = self.make_document(doc_a1.doc_id, 'test:1|other:1', - '{"a":"42"}') - doc_a3 = self.make_document(doc_a1.doc_id, 'test:2|other:1', "{}") - state, _ = self.db._put_doc_if_newer( - doc_a2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual(state, 'inserted') - state, _ = self.db._put_doc_if_newer( - doc_a1b1, save_conflict=True, replica_uid='r', replica_gen=2, - replica_trans_id='foo2') - self.assertEqual(state, 'conflicted') - state, _ = self.db._put_doc_if_newer( - doc_a3, save_conflict=True, replica_uid='r', replica_gen=3, - replica_trans_id='foo3') - self.assertEqual(state, 'inserted') - self.assertFalse(self.db.get_doc(doc_a1.doc_id).has_conflicts) - - def test_put_doc_if_newer_autoresolve_3(self): - doc_a1 = self.db.create_doc_from_json(simple_doc) - doc_a1b1 = self.make_document(doc_a1.doc_id, 'test:1|other:1', "{}") - doc_a2 = self.make_document(doc_a1.doc_id, 'test:2', '{"a":"42"}') - doc_a3 = self.make_document(doc_a1.doc_id, 'test:3', "{}") - state, _ = self.db._put_doc_if_newer( - doc_a1b1, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual(state, 'inserted') - state, _ = self.db._put_doc_if_newer( - doc_a2, save_conflict=True, replica_uid='r', replica_gen=2, - replica_trans_id='foo2') - self.assertEqual(state, 'conflicted') - state, _ = self.db._put_doc_if_newer( - doc_a3, save_conflict=True, replica_uid='r', replica_gen=3, - replica_trans_id='foo3') - self.assertEqual(state, 'superseded') - doc = self.db.get_doc(doc_a1.doc_id, True) - self.assertFalse(doc.has_conflicts) - rev = vectorclock.VectorClockRev(doc.rev) - rev_a3 = vectorclock.VectorClockRev('test:3') - rev_a1b1 = vectorclock.VectorClockRev('test:1|other:1') - self.assertTrue(rev.is_newer(rev_a3)) - self.assertTrue('test:4' in doc.rev) # locally increased - self.assertTrue(rev.is_newer(rev_a1b1)) - - def test_put_doc_if_newer_autoresolve_4(self): - doc_a1 = self.db.create_doc_from_json(simple_doc) - doc_a1b1 = self.make_document(doc_a1.doc_id, 'test:1|other:1', None) - doc_a2 = self.make_document(doc_a1.doc_id, 'test:2', '{"a":"42"}') - doc_a3 = self.make_document(doc_a1.doc_id, 'test:3', None) - state, _ = self.db._put_doc_if_newer( - doc_a1b1, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertEqual(state, 'inserted') - state, _ = self.db._put_doc_if_newer( - doc_a2, save_conflict=True, replica_uid='r', replica_gen=2, - replica_trans_id='foo2') - self.assertEqual(state, 'conflicted') - state, _ = self.db._put_doc_if_newer( - doc_a3, save_conflict=True, replica_uid='r', replica_gen=3, - replica_trans_id='foo3') - self.assertEqual(state, 'superseded') - doc = self.db.get_doc(doc_a1.doc_id, True) - self.assertFalse(doc.has_conflicts) - rev = vectorclock.VectorClockRev(doc.rev) - rev_a3 = vectorclock.VectorClockRev('test:3') - rev_a1b1 = vectorclock.VectorClockRev('test:1|other:1') - self.assertTrue(rev.is_newer(rev_a3)) - self.assertTrue('test:4' in doc.rev) # locally increased - self.assertTrue(rev.is_newer(rev_a1b1)) - - def test_put_refuses_to_update_conflicted(self): - doc1 = self.db.create_doc_from_json(simple_doc) - content2 = '{"key": "altval"}' - doc2 = self.make_document(doc1.doc_id, 'altrev:1', content2) - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertGetDoc(self.db, doc1.doc_id, doc2.rev, content2, True) - content3 = '{"key": "local"}' - doc2.set_json(content3) - self.assertRaises(errors.ConflictedDoc, self.db.put_doc, doc2) - - def test_delete_refuses_for_conflicted(self): - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.make_document(doc1.doc_id, 'altrev:1', nested_doc) - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertGetDoc(self.db, doc2.doc_id, doc2.rev, nested_doc, True) - self.assertRaises(errors.ConflictedDoc, self.db.delete_doc, doc2) - - -@skip("Skiping tests imported from U1DB.") -class DatabaseIndexTests(tests.DatabaseBaseTests): - - scenarios = tests.LOCAL_DATABASES_SCENARIOS - - def assertParseError(self, definition): - self.db.create_doc_from_json(nested_doc) - self.assertRaises( - errors.IndexDefinitionParseError, self.db.create_index, 'idx', - definition) - - def assertIndexCreatable(self, definition): - name = "idx" - self.db.create_doc_from_json(nested_doc) - self.db.create_index(name, definition) - self.assertEqual( - [(name, [definition])], self.db.list_indexes()) - - def test_create_index(self): - self.db.create_index('test-idx', 'name') - self.assertEqual([('test-idx', ['name'])], - self.db.list_indexes()) - - def test_create_index_on_non_ascii_field_name(self): - doc = self.db.create_doc_from_json(json.dumps({u'\xe5': 'value'})) - self.db.create_index('test-idx', u'\xe5') - self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) - - def test_list_indexes_with_non_ascii_field_names(self): - self.db.create_index('test-idx', u'\xe5') - self.assertEqual( - [('test-idx', [u'\xe5'])], self.db.list_indexes()) - - def test_create_index_evaluates_it(self): - doc = self.db.create_doc_from_json(simple_doc) - self.db.create_index('test-idx', 'key') - self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) - - def test_wildcard_matches_unicode_value(self): - doc = self.db.create_doc_from_json(json.dumps({"key": u"valu\xe5"})) - self.db.create_index('test-idx', 'key') - self.assertEqual([doc], self.db.get_from_index('test-idx', '*')) - - def test_retrieve_unicode_value_from_index(self): - doc = self.db.create_doc_from_json(json.dumps({"key": u"valu\xe5"})) - self.db.create_index('test-idx', 'key') - self.assertEqual( - [doc], self.db.get_from_index('test-idx', u"valu\xe5")) - - def test_create_index_fails_if_name_taken(self): - self.db.create_index('test-idx', 'key') - self.assertRaises(errors.IndexNameTakenError, - self.db.create_index, - 'test-idx', 'stuff') - - def test_create_index_does_not_fail_if_name_taken_with_same_index(self): - self.db.create_index('test-idx', 'key') - self.db.create_index('test-idx', 'key') - self.assertEqual([('test-idx', ['key'])], self.db.list_indexes()) - - def test_create_index_does_not_duplicate_indexed_fields(self): - self.db.create_doc_from_json(simple_doc) - self.db.create_index('test-idx', 'key') - self.db.delete_index('test-idx') - self.db.create_index('test-idx', 'key') - self.assertEqual(1, len(self.db.get_from_index('test-idx', 'value'))) - - def test_delete_index_does_not_remove_fields_from_other_indexes(self): - self.db.create_doc_from_json(simple_doc) - self.db.create_index('test-idx', 'key') - self.db.create_index('test-idx2', 'key') - self.db.delete_index('test-idx') - self.assertEqual(1, len(self.db.get_from_index('test-idx2', 'value'))) - - def test_create_index_after_deleting_document(self): - doc = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(simple_doc) - self.db.delete_doc(doc2) - self.db.create_index('test-idx', 'key') - self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) - - def test_delete_index(self): - self.db.create_index('test-idx', 'key') - self.assertEqual([('test-idx', ['key'])], self.db.list_indexes()) - self.db.delete_index('test-idx') - self.assertEqual([], self.db.list_indexes()) - - def test_create_adds_to_index(self): - self.db.create_index('test-idx', 'key') - doc = self.db.create_doc_from_json(simple_doc) - self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) - - def test_get_from_index_unmatched(self): - self.db.create_doc_from_json(simple_doc) - self.db.create_index('test-idx', 'key') - self.assertEqual([], self.db.get_from_index('test-idx', 'novalue')) - - def test_create_index_multiple_exact_matches(self): - doc = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(simple_doc) - self.db.create_index('test-idx', 'key') - self.assertEqual( - sorted([doc, doc2]), - sorted(self.db.get_from_index('test-idx', 'value'))) - - def test_get_from_index(self): - doc = self.db.create_doc_from_json(simple_doc) - self.db.create_index('test-idx', 'key') - self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) - - def test_get_from_index_multi(self): - content = '{"key": "value", "key2": "value2"}' - doc = self.db.create_doc_from_json(content) - self.db.create_index('test-idx', 'key', 'key2') - self.assertEqual( - [doc], self.db.get_from_index('test-idx', 'value', 'value2')) - - def test_get_from_index_multi_list(self): - doc = self.db.create_doc_from_json( - '{"key": "value", "key2": ["value2-1", "value2-2", "value2-3"]}') - self.db.create_index('test-idx', 'key', 'key2') - self.assertEqual( - [doc], self.db.get_from_index('test-idx', 'value', 'value2-1')) - self.assertEqual( - [doc], self.db.get_from_index('test-idx', 'value', 'value2-2')) - self.assertEqual( - [doc], self.db.get_from_index('test-idx', 'value', 'value2-3')) - self.assertEqual( - [('value', 'value2-1'), ('value', 'value2-2'), - ('value', 'value2-3')], - sorted(self.db.get_index_keys('test-idx'))) - - def test_get_from_index_sees_conflicts(self): - doc = self.db.create_doc_from_json(simple_doc) - self.db.create_index('test-idx', 'key', 'key2') - alt_doc = self.make_document( - doc.doc_id, 'alternate:1', - '{"key": "value", "key2": ["value2-1", "value2-2", "value2-3"]}') - self.db._put_doc_if_newer( - alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - docs = self.db.get_from_index('test-idx', 'value', 'value2-1') - self.assertTrue(docs[0].has_conflicts) - - def test_get_index_keys_multi_list_list(self): - self.db.create_doc_from_json( - '{"key": "value1-1 value1-2 value1-3", ' - '"key2": ["value2-1", "value2-2", "value2-3"]}') - self.db.create_index('test-idx', 'split_words(key)', 'key2') - self.assertEqual( - [(u'value1-1', u'value2-1'), (u'value1-1', u'value2-2'), - (u'value1-1', u'value2-3'), (u'value1-2', u'value2-1'), - (u'value1-2', u'value2-2'), (u'value1-2', u'value2-3'), - (u'value1-3', u'value2-1'), (u'value1-3', u'value2-2'), - (u'value1-3', u'value2-3')], - sorted(self.db.get_index_keys('test-idx'))) - - def test_get_from_index_multi_ordered(self): - doc1 = self.db.create_doc_from_json( - '{"key": "value3", "key2": "value4"}') - doc2 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value3"}') - doc3 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value2"}') - doc4 = self.db.create_doc_from_json( - '{"key": "value1", "key2": "value1"}') - self.db.create_index('test-idx', 'key', 'key2') - self.assertEqual( - [doc4, doc3, doc2, doc1], - self.db.get_from_index('test-idx', 'v*', '*')) - - def test_get_range_from_index_start_end(self): - doc1 = self.db.create_doc_from_json('{"key": "value3"}') - doc2 = self.db.create_doc_from_json('{"key": "value2"}') - self.db.create_doc_from_json('{"key": "value4"}') - self.db.create_doc_from_json('{"key": "value1"}') - self.db.create_index('test-idx', 'key') - self.assertEqual( - [doc2, doc1], - self.db.get_range_from_index('test-idx', 'value2', 'value3')) - - def test_get_range_from_index_start(self): - doc1 = self.db.create_doc_from_json('{"key": "value3"}') - doc2 = self.db.create_doc_from_json('{"key": "value2"}') - doc3 = self.db.create_doc_from_json('{"key": "value4"}') - self.db.create_doc_from_json('{"key": "value1"}') - self.db.create_index('test-idx', 'key') - self.assertEqual( - [doc2, doc1, doc3], - self.db.get_range_from_index('test-idx', 'value2')) - - def test_get_range_from_index_sees_conflicts(self): - doc = self.db.create_doc_from_json(simple_doc) - self.db.create_index('test-idx', 'key') - alt_doc = self.make_document( - doc.doc_id, 'alternate:1', '{"key": "valuedepalue"}') - self.db._put_doc_if_newer( - alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - docs = self.db.get_range_from_index('test-idx', 'a') - self.assertTrue(docs[0].has_conflicts) - - def test_get_range_from_index_end(self): - self.db.create_doc_from_json('{"key": "value3"}') - doc2 = self.db.create_doc_from_json('{"key": "value2"}') - self.db.create_doc_from_json('{"key": "value4"}') - doc4 = self.db.create_doc_from_json('{"key": "value1"}') - self.db.create_index('test-idx', 'key') - self.assertEqual( - [doc4, doc2], - self.db.get_range_from_index('test-idx', None, 'value2')) - - def test_get_wildcard_range_from_index_start(self): - doc1 = self.db.create_doc_from_json('{"key": "value4"}') - doc2 = self.db.create_doc_from_json('{"key": "value23"}') - doc3 = self.db.create_doc_from_json('{"key": "value2"}') - doc4 = self.db.create_doc_from_json('{"key": "value22"}') - self.db.create_doc_from_json('{"key": "value1"}') - self.db.create_index('test-idx', 'key') - self.assertEqual( - [doc3, doc4, doc2, doc1], - self.db.get_range_from_index('test-idx', 'value2*')) - - def test_get_wildcard_range_from_index_end(self): - self.db.create_doc_from_json('{"key": "value4"}') - doc2 = self.db.create_doc_from_json('{"key": "value23"}') - doc3 = self.db.create_doc_from_json('{"key": "value2"}') - doc4 = self.db.create_doc_from_json('{"key": "value22"}') - doc5 = self.db.create_doc_from_json('{"key": "value1"}') - self.db.create_index('test-idx', 'key') - self.assertEqual( - [doc5, doc3, doc4, doc2], - self.db.get_range_from_index('test-idx', None, 'value2*')) - - def test_get_wildcard_range_from_index_start_end(self): - self.db.create_doc_from_json('{"key": "a"}') - self.db.create_doc_from_json('{"key": "boo3"}') - doc3 = self.db.create_doc_from_json('{"key": "catalyst"}') - doc4 = self.db.create_doc_from_json('{"key": "whaever"}') - self.db.create_doc_from_json('{"key": "zerg"}') - self.db.create_index('test-idx', 'key') - self.assertEqual( - [doc3, doc4], - self.db.get_range_from_index('test-idx', 'cat*', 'zap*')) - - def test_get_range_from_index_multi_column_start_end(self): - self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') - doc2 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value3"}') - doc3 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value2"}') - self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') - self.db.create_index('test-idx', 'key', 'key2') - self.assertEqual( - [doc3, doc2], - self.db.get_range_from_index( - 'test-idx', ('value2', 'value2'), ('value2', 'value3'))) - - def test_get_range_from_index_multi_column_start(self): - doc1 = self.db.create_doc_from_json( - '{"key": "value3", "key2": "value4"}') - doc2 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value3"}') - self.db.create_doc_from_json('{"key": "value2", "key2": "value2"}') - self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') - self.db.create_index('test-idx', 'key', 'key2') - self.assertEqual( - [doc2, doc1], - self.db.get_range_from_index('test-idx', ('value2', 'value3'))) - - def test_get_range_from_index_multi_column_end(self): - self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') - doc2 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value3"}') - doc3 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value2"}') - doc4 = self.db.create_doc_from_json( - '{"key": "value1", "key2": "value1"}') - self.db.create_index('test-idx', 'key', 'key2') - self.assertEqual( - [doc4, doc3, doc2], - self.db.get_range_from_index( - 'test-idx', None, ('value2', 'value3'))) - - def test_get_wildcard_range_from_index_multi_column_start(self): - doc1 = self.db.create_doc_from_json( - '{"key": "value3", "key2": "value4"}') - doc2 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value23"}') - doc3 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value2"}') - self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') - self.db.create_index('test-idx', 'key', 'key2') - self.assertEqual( - [doc3, doc2, doc1], - self.db.get_range_from_index('test-idx', ('value2', 'value2*'))) - - def test_get_wildcard_range_from_index_multi_column_end(self): - self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') - doc2 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value23"}') - doc3 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value2"}') - doc4 = self.db.create_doc_from_json( - '{"key": "value1", "key2": "value1"}') - self.db.create_index('test-idx', 'key', 'key2') - self.assertEqual( - [doc4, doc3, doc2], - self.db.get_range_from_index( - 'test-idx', None, ('value2', 'value2*'))) - - def test_get_glob_range_from_index_multi_column_start(self): - doc1 = self.db.create_doc_from_json( - '{"key": "value3", "key2": "value4"}') - doc2 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value23"}') - self.db.create_doc_from_json('{"key": "value1", "key2": "value2"}') - self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') - self.db.create_index('test-idx', 'key', 'key2') - self.assertEqual( - [doc2, doc1], - self.db.get_range_from_index('test-idx', ('value2', '*'))) - - def test_get_glob_range_from_index_multi_column_end(self): - self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') - doc2 = self.db.create_doc_from_json( - '{"key": "value2", "key2": "value23"}') - doc3 = self.db.create_doc_from_json( - '{"key": "value1", "key2": "value2"}') - doc4 = self.db.create_doc_from_json( - '{"key": "value1", "key2": "value1"}') - self.db.create_index('test-idx', 'key', 'key2') - self.assertEqual( - [doc4, doc3, doc2], - self.db.get_range_from_index('test-idx', None, ('value2', '*'))) - - def test_get_range_from_index_illegal_wildcard_order(self): - self.db.create_index('test-idx', 'k1', 'k2') - self.assertRaises( - errors.InvalidGlobbing, - self.db.get_range_from_index, 'test-idx', ('*', 'v2')) - - def test_get_range_from_index_illegal_glob_after_wildcard(self): - self.db.create_index('test-idx', 'k1', 'k2') - self.assertRaises( - errors.InvalidGlobbing, - self.db.get_range_from_index, 'test-idx', ('*', 'v*')) - - def test_get_range_from_index_illegal_wildcard_order_end(self): - self.db.create_index('test-idx', 'k1', 'k2') - self.assertRaises( - errors.InvalidGlobbing, - self.db.get_range_from_index, 'test-idx', None, ('*', 'v2')) - - def test_get_range_from_index_illegal_glob_after_wildcard_end(self): - self.db.create_index('test-idx', 'k1', 'k2') - self.assertRaises( - errors.InvalidGlobbing, - self.db.get_range_from_index, 'test-idx', None, ('*', 'v*')) - - def test_get_from_index_fails_if_no_index(self): - self.assertRaises( - errors.IndexDoesNotExist, self.db.get_from_index, 'foo') - - def test_get_index_keys_fails_if_no_index(self): - self.assertRaises(errors.IndexDoesNotExist, - self.db.get_index_keys, - 'foo') - - def test_get_index_keys_works_if_no_docs(self): - self.db.create_index('test-idx', 'key') - self.assertEqual([], self.db.get_index_keys('test-idx')) - - def test_put_updates_index(self): - doc = self.db.create_doc_from_json(simple_doc) - self.db.create_index('test-idx', 'key') - new_content = '{"key": "altval"}' - doc.set_json(new_content) - self.db.put_doc(doc) - self.assertEqual([], self.db.get_from_index('test-idx', 'value')) - self.assertEqual([doc], self.db.get_from_index('test-idx', 'altval')) - - def test_delete_updates_index(self): - doc = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(simple_doc) - self.db.create_index('test-idx', 'key') - self.assertEqual( - sorted([doc, doc2]), - sorted(self.db.get_from_index('test-idx', 'value'))) - self.db.delete_doc(doc) - self.assertEqual([doc2], self.db.get_from_index('test-idx', 'value')) - - def test_get_from_index_illegal_number_of_entries(self): - self.db.create_index('test-idx', 'k1', 'k2') - self.assertRaises( - errors.InvalidValueForIndex, self.db.get_from_index, 'test-idx') - self.assertRaises( - errors.InvalidValueForIndex, - self.db.get_from_index, 'test-idx', 'v1') - self.assertRaises( - errors.InvalidValueForIndex, - self.db.get_from_index, 'test-idx', 'v1', 'v2', 'v3') - - def test_get_from_index_illegal_wildcard_order(self): - self.db.create_index('test-idx', 'k1', 'k2') - self.assertRaises( - errors.InvalidGlobbing, - self.db.get_from_index, 'test-idx', '*', 'v2') - - def test_get_from_index_illegal_glob_after_wildcard(self): - self.db.create_index('test-idx', 'k1', 'k2') - self.assertRaises( - errors.InvalidGlobbing, - self.db.get_from_index, 'test-idx', '*', 'v*') - - def test_get_all_from_index(self): - self.db.create_index('test-idx', 'key') - doc1 = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(nested_doc) - # This one should not be in the index - self.db.create_doc_from_json('{"no": "key"}') - diff_value_doc = '{"key": "diff value"}' - doc4 = self.db.create_doc_from_json(diff_value_doc) - # This is essentially a 'prefix' match, but we match every entry. - self.assertEqual( - sorted([doc1, doc2, doc4]), - sorted(self.db.get_from_index('test-idx', '*'))) - - def test_get_all_from_index_ordered(self): - self.db.create_index('test-idx', 'key') - doc1 = self.db.create_doc_from_json('{"key": "value x"}') - doc2 = self.db.create_doc_from_json('{"key": "value b"}') - doc3 = self.db.create_doc_from_json('{"key": "value a"}') - doc4 = self.db.create_doc_from_json('{"key": "value m"}') - # This is essentially a 'prefix' match, but we match every entry. - self.assertEqual( - [doc3, doc2, doc4, doc1], self.db.get_from_index('test-idx', '*')) - - def test_put_updates_when_adding_key(self): - doc = self.db.create_doc_from_json("{}") - self.db.create_index('test-idx', 'key') - self.assertEqual([], self.db.get_from_index('test-idx', '*')) - doc.set_json(simple_doc) - self.db.put_doc(doc) - self.assertEqual([doc], self.db.get_from_index('test-idx', '*')) - - def test_get_from_index_empty_string(self): - self.db.create_index('test-idx', 'key') - doc1 = self.db.create_doc_from_json(simple_doc) - content2 = '{"key": ""}' - doc2 = self.db.create_doc_from_json(content2) - self.assertEqual([doc2], self.db.get_from_index('test-idx', '')) - # Empty string matches the wildcard. - self.assertEqual( - sorted([doc1, doc2]), - sorted(self.db.get_from_index('test-idx', '*'))) - - def test_get_from_index_not_null(self): - self.db.create_index('test-idx', 'key') - doc1 = self.db.create_doc_from_json(simple_doc) - self.db.create_doc_from_json('{"key": null}') - self.assertEqual([doc1], self.db.get_from_index('test-idx', '*')) - - def test_get_partial_from_index(self): - content1 = '{"k1": "v1", "k2": "v2"}' - content2 = '{"k1": "v1", "k2": "x2"}' - content3 = '{"k1": "v1", "k2": "y2"}' - # doc4 has a different k1 value, so it doesn't match the prefix. - content4 = '{"k1": "NN", "k2": "v2"}' - doc1 = self.db.create_doc_from_json(content1) - doc2 = self.db.create_doc_from_json(content2) - doc3 = self.db.create_doc_from_json(content3) - self.db.create_doc_from_json(content4) - self.db.create_index('test-idx', 'k1', 'k2') - self.assertEqual( - sorted([doc1, doc2, doc3]), - sorted(self.db.get_from_index('test-idx', "v1", "*"))) - - def test_get_glob_match(self): - # Note: the exact glob syntax is probably subject to change - content1 = '{"k1": "v1", "k2": "v1"}' - content2 = '{"k1": "v1", "k2": "v2"}' - content3 = '{"k1": "v1", "k2": "v3"}' - # doc4 has a different k2 prefix value, so it doesn't match - content4 = '{"k1": "v1", "k2": "ZZ"}' - self.db.create_index('test-idx', 'k1', 'k2') - doc1 = self.db.create_doc_from_json(content1) - doc2 = self.db.create_doc_from_json(content2) - doc3 = self.db.create_doc_from_json(content3) - self.db.create_doc_from_json(content4) - self.assertEqual( - sorted([doc1, doc2, doc3]), - sorted(self.db.get_from_index('test-idx', "v1", "v*"))) - - def test_nested_index(self): - doc = self.db.create_doc_from_json(nested_doc) - self.db.create_index('test-idx', 'sub.doc') - self.assertEqual( - [doc], self.db.get_from_index('test-idx', 'underneath')) - doc2 = self.db.create_doc_from_json(nested_doc) - self.assertEqual( - sorted([doc, doc2]), - sorted(self.db.get_from_index('test-idx', 'underneath'))) - - def test_nested_nonexistent(self): - self.db.create_doc_from_json(nested_doc) - # sub exists, but sub.foo does not: - self.db.create_index('test-idx', 'sub.foo') - self.assertEqual([], self.db.get_from_index('test-idx', '*')) - - def test_nested_nonexistent2(self): - self.db.create_doc_from_json(nested_doc) - self.db.create_index('test-idx', 'sub.foo.bar.baz.qux.fnord') - self.assertEqual([], self.db.get_from_index('test-idx', '*')) - - def test_nested_traverses_lists(self): - # subpath finds dicts in list - doc = self.db.create_doc_from_json( - '{"foo": [{"zap": "bar"}, {"zap": "baz"}]}') - # subpath only finds dicts in list - self.db.create_doc_from_json('{"foo": ["zap", "baz"]}') - self.db.create_index('test-idx', 'foo.zap') - self.assertEqual([doc], self.db.get_from_index('test-idx', 'bar')) - self.assertEqual([doc], self.db.get_from_index('test-idx', 'baz')) - - def test_nested_list_traversal(self): - # subpath finds dicts in list - doc = self.db.create_doc_from_json( - '{"foo": [{"zap": [{"qux": "fnord"}, {"qux": "zombo"}]},' - '{"zap": "baz"}]}') - # subpath only finds dicts in list - self.db.create_index('test-idx', 'foo.zap.qux') - self.assertEqual([doc], self.db.get_from_index('test-idx', 'fnord')) - self.assertEqual([doc], self.db.get_from_index('test-idx', 'zombo')) - - def test_index_list1(self): - self.db.create_index("index", "name") - content = '{"name": ["foo", "bar"]}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "bar") - self.assertEqual([doc], rows) - - def test_index_list2(self): - self.db.create_index("index", "name") - content = '{"name": ["foo", "bar"]}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "foo") - self.assertEqual([doc], rows) - - def test_get_from_index_case_sensitive(self): - self.db.create_index('test-idx', 'key') - doc1 = self.db.create_doc_from_json(simple_doc) - self.assertEqual([], self.db.get_from_index('test-idx', 'V*')) - self.assertEqual([doc1], self.db.get_from_index('test-idx', 'v*')) - - def test_get_from_index_illegal_glob_before_value(self): - self.db.create_index('test-idx', 'k1', 'k2') - self.assertRaises( - errors.InvalidGlobbing, - self.db.get_from_index, 'test-idx', 'v*', 'v2') - - def test_get_from_index_illegal_glob_after_glob(self): - self.db.create_index('test-idx', 'k1', 'k2') - self.assertRaises( - errors.InvalidGlobbing, - self.db.get_from_index, 'test-idx', 'v*', 'v*') - - def test_get_from_index_with_sql_wildcards(self): - self.db.create_index('test-idx', 'key') - content1 = '{"key": "va%lue"}' - content2 = '{"key": "value"}' - content3 = '{"key": "va_lue"}' - doc1 = self.db.create_doc_from_json(content1) - self.db.create_doc_from_json(content2) - doc3 = self.db.create_doc_from_json(content3) - # The '%' in the search should be treated literally, not as a sql - # globbing character. - self.assertEqual([doc1], self.db.get_from_index('test-idx', 'va%*')) - # Same for '_' - self.assertEqual([doc3], self.db.get_from_index('test-idx', 'va_*')) - - def test_get_from_index_with_lower(self): - self.db.create_index("index", "lower(name)") - content = '{"name": "Foo"}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "foo") - self.assertEqual([doc], rows) - - def test_get_from_index_with_lower_matches_same_case(self): - self.db.create_index("index", "lower(name)") - content = '{"name": "foo"}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "foo") - self.assertEqual([doc], rows) - - def test_index_lower_doesnt_match_different_case(self): - self.db.create_index("index", "lower(name)") - content = '{"name": "Foo"}' - self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "Foo") - self.assertEqual([], rows) - - def test_index_lower_doesnt_match_other_index(self): - self.db.create_index("index", "lower(name)") - self.db.create_index("other_index", "name") - content = '{"name": "Foo"}' - self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "Foo") - self.assertEqual(0, len(rows)) - - def test_index_split_words_match_first(self): - self.db.create_index("index", "split_words(name)") - content = '{"name": "foo bar"}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "foo") - self.assertEqual([doc], rows) - - def test_index_split_words_match_second(self): - self.db.create_index("index", "split_words(name)") - content = '{"name": "foo bar"}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "bar") - self.assertEqual([doc], rows) - - def test_index_split_words_match_both(self): - self.db.create_index("index", "split_words(name)") - content = '{"name": "foo foo"}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "foo") - self.assertEqual([doc], rows) - - def test_index_split_words_double_space(self): - self.db.create_index("index", "split_words(name)") - content = '{"name": "foo bar"}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "bar") - self.assertEqual([doc], rows) - - def test_index_split_words_leading_space(self): - self.db.create_index("index", "split_words(name)") - content = '{"name": " foo bar"}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "foo") - self.assertEqual([doc], rows) - - def test_index_split_words_trailing_space(self): - self.db.create_index("index", "split_words(name)") - content = '{"name": "foo bar "}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "bar") - self.assertEqual([doc], rows) - - def test_get_from_index_with_number(self): - self.db.create_index("index", "number(foo, 5)") - content = '{"foo": 12}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "00012") - self.assertEqual([doc], rows) - - def test_get_from_index_with_number_bigger_than_padding(self): - self.db.create_index("index", "number(foo, 5)") - content = '{"foo": 123456}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "123456") - self.assertEqual([doc], rows) - - def test_number_mapping_ignores_non_numbers(self): - self.db.create_index("index", "number(foo, 5)") - content = '{"foo": 56}' - doc1 = self.db.create_doc_from_json(content) - content = '{"foo": "this is not a maigret painting"}' - self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "*") - self.assertEqual([doc1], rows) - - def test_get_from_index_with_bool(self): - self.db.create_index("index", "bool(foo)") - content = '{"foo": true}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "1") - self.assertEqual([doc], rows) - - def test_get_from_index_with_bool_false(self): - self.db.create_index("index", "bool(foo)") - content = '{"foo": false}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "0") - self.assertEqual([doc], rows) - - def test_get_from_index_with_non_bool(self): - self.db.create_index("index", "bool(foo)") - content = '{"foo": 42}' - self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "*") - self.assertEqual([], rows) - - def test_get_from_index_with_combine(self): - self.db.create_index("index", "combine(foo, bar)") - content = '{"foo": "value1", "bar": "value2"}' - doc = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "value1") - self.assertEqual([doc], rows) - rows = self.db.get_from_index("index", "value2") - self.assertEqual([doc], rows) - - def test_get_complex_combine(self): - self.db.create_index( - "index", "combine(number(foo, 5), lower(bar), split_words(baz))") - content = '{"foo": 12, "bar": "ALLCAPS", "baz": "qux nox"}' - doc = self.db.create_doc_from_json(content) - content = '{"foo": "not a number", "bar": "something"}' - doc2 = self.db.create_doc_from_json(content) - rows = self.db.get_from_index("index", "00012") - self.assertEqual([doc], rows) - rows = self.db.get_from_index("index", "allcaps") - self.assertEqual([doc], rows) - rows = self.db.get_from_index("index", "nox") - self.assertEqual([doc], rows) - rows = self.db.get_from_index("index", "something") - self.assertEqual([doc2], rows) - - def test_get_index_keys_from_index(self): - self.db.create_index('test-idx', 'key') - content1 = '{"key": "value1"}' - content2 = '{"key": "value2"}' - content3 = '{"key": "value2"}' - self.db.create_doc_from_json(content1) - self.db.create_doc_from_json(content2) - self.db.create_doc_from_json(content3) - self.assertEqual( - [('value1',), ('value2',)], - sorted(self.db.get_index_keys('test-idx'))) - - def test_get_index_keys_from_multicolumn_index(self): - self.db.create_index('test-idx', 'key1', 'key2') - content1 = '{"key1": "value1", "key2": "val2-1"}' - content2 = '{"key1": "value2", "key2": "val2-2"}' - content3 = '{"key1": "value2", "key2": "val2-2"}' - content4 = '{"key1": "value2", "key2": "val3"}' - self.db.create_doc_from_json(content1) - self.db.create_doc_from_json(content2) - self.db.create_doc_from_json(content3) - self.db.create_doc_from_json(content4) - self.assertEqual([ - ('value1', 'val2-1'), - ('value2', 'val2-2'), - ('value2', 'val3')], - sorted(self.db.get_index_keys('test-idx'))) - - def test_empty_expr(self): - self.assertParseError('') - - def test_nested_unknown_operation(self): - self.assertParseError('unknown_operation(field1)') - - def test_parse_missing_close_paren(self): - self.assertParseError("lower(a") - - def test_parse_trailing_close_paren(self): - self.assertParseError("lower(ab))") - - def test_parse_trailing_chars(self): - self.assertParseError("lower(ab)adsf") - - def test_parse_empty_op(self): - self.assertParseError("(ab)") - - def test_parse_top_level_commas(self): - self.assertParseError("a, b") - - def test_invalid_field_name(self): - self.assertParseError("a.") - - def test_invalid_inner_field_name(self): - self.assertParseError("lower(a.)") - - def test_gobbledigook(self): - self.assertParseError("(@#@cc @#!*DFJSXV(()jccd") - - def test_leading_space(self): - self.assertIndexCreatable(" lower(a)") - - def test_trailing_space(self): - self.assertIndexCreatable("lower(a) ") - - def test_spaces_before_open_paren(self): - self.assertIndexCreatable("lower (a)") - - def test_spaces_after_open_paren(self): - self.assertIndexCreatable("lower( a)") - - def test_spaces_before_close_paren(self): - self.assertIndexCreatable("lower(a )") - - def test_spaces_before_comma(self): - self.assertIndexCreatable("combine(a , b , c)") - - def test_spaces_after_comma(self): - self.assertIndexCreatable("combine(a, b, c)") - - def test_all_together_now(self): - self.assertParseError(' (a) ') - - def test_all_together_now2(self): - self.assertParseError('combine(lower(x)x,foo)') - - -@skip("Skiping tests imported from U1DB.") -class PythonBackendTests(tests.DatabaseBaseTests): - - def setUp(self): - super(PythonBackendTests, self).setUp() - self.simple_doc = json.loads(simple_doc) - - def test_create_doc_with_factory(self): - self.db.set_document_factory(TestAlternativeDocument) - doc = self.db.create_doc(self.simple_doc, doc_id='my_doc_id') - self.assertTrue(isinstance(doc, TestAlternativeDocument)) - - def test_get_doc_after_put_with_factory(self): - doc = self.db.create_doc(self.simple_doc, doc_id='my_doc_id') - self.db.set_document_factory(TestAlternativeDocument) - result = self.db.get_doc('my_doc_id') - self.assertTrue(isinstance(result, TestAlternativeDocument)) - self.assertEqual(doc.doc_id, result.doc_id) - self.assertEqual(doc.rev, result.rev) - self.assertEqual(doc.get_json(), result.get_json()) - self.assertEqual(False, result.has_conflicts) - - def test_get_doc_nonexisting_with_factory(self): - self.db.set_document_factory(TestAlternativeDocument) - self.assertIs(None, self.db.get_doc('non-existing')) - - def test_get_all_docs_with_factory(self): - self.db.set_document_factory(TestAlternativeDocument) - self.db.create_doc(self.simple_doc) - self.assertTrue(isinstance( - list(self.db.get_all_docs()[1])[0], TestAlternativeDocument)) - - def test_get_docs_conflicted_with_factory(self): - self.db.set_document_factory(TestAlternativeDocument) - doc1 = self.db.create_doc(self.simple_doc) - doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) - self.db._put_doc_if_newer( - doc2, save_conflict=True, replica_uid='r', replica_gen=1, - replica_trans_id='foo') - self.assertTrue( - isinstance( - list(self.db.get_docs([doc1.doc_id]))[0], - TestAlternativeDocument)) - - def test_get_from_index_with_factory(self): - self.db.set_document_factory(TestAlternativeDocument) - self.db.create_doc(self.simple_doc) - self.db.create_index('test-idx', 'key') - self.assertTrue( - isinstance( - self.db.get_from_index('test-idx', 'value')[0], - TestAlternativeDocument)) - - def test_sync_exchange_updates_indexes(self): - doc = self.db.create_doc(self.simple_doc) - self.db.create_index('test-idx', 'key') - new_content = '{"key": "altval"}' - other_rev = 'test:1|z:2' - st = self.db.get_sync_target() - - def ignore(doc_id, doc_rev, doc): - pass - - doc_other = self.make_document(doc.doc_id, other_rev, new_content) - docs_by_gen = [(doc_other, 10, 'T-sid')] - st.sync_exchange( - docs_by_gen, 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=ignore) - self.assertGetDoc(self.db, doc.doc_id, other_rev, new_content, False) - self.assertEqual( - [doc_other], self.db.get_from_index('test-idx', 'altval')) - self.assertEqual([], self.db.get_from_index('test-idx', 'value')) - - -# Use a custom loader to apply the scenarios at load time. -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_document.py b/common/src/leap/soledad/common/tests/u1db_tests/test_document.py deleted file mode 100644 index 4e8bcaf9..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_document.py +++ /dev/null @@ -1,153 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# Copyright 2016 LEAP Encryption Access Project -# -# This file is part of leap.soledad.common -# -# leap.soledad.common is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see . -from unittest import skip - -from leap.soledad.common.l2db import errors - -from leap.soledad.common.tests import u1db_tests as tests - - -@skip("Skiping tests imported from U1DB.") -class TestDocument(tests.TestCase): - - scenarios = ([( - 'py', {'make_document_for_test': tests.make_document_for_test})]) # + - # tests.C_DATABASE_SCENARIOS) - - def test_create_doc(self): - doc = self.make_document('doc-id', 'uid:1', tests.simple_doc) - self.assertEqual('doc-id', doc.doc_id) - self.assertEqual('uid:1', doc.rev) - self.assertEqual(tests.simple_doc, doc.get_json()) - self.assertFalse(doc.has_conflicts) - - def test__repr__(self): - doc = self.make_document('doc-id', 'uid:1', tests.simple_doc) - self.assertEqual( - '%s(doc-id, uid:1, \'{"key": "value"}\')' - % (doc.__class__.__name__,), - repr(doc)) - - def test__repr__conflicted(self): - doc = self.make_document('doc-id', 'uid:1', tests.simple_doc, - has_conflicts=True) - self.assertEqual( - '%s(doc-id, uid:1, conflicted, \'{"key": "value"}\')' - % (doc.__class__.__name__,), - repr(doc)) - - def test__lt__(self): - doc_a = self.make_document('a', 'b', '{}') - doc_b = self.make_document('b', 'b', '{}') - self.assertTrue(doc_a < doc_b) - self.assertTrue(doc_b > doc_a) - doc_aa = self.make_document('a', 'a', '{}') - self.assertTrue(doc_aa < doc_a) - - def test__eq__(self): - doc_a = self.make_document('a', 'b', '{}') - doc_b = self.make_document('a', 'b', '{}') - self.assertTrue(doc_a == doc_b) - doc_b = self.make_document('a', 'b', '{}', has_conflicts=True) - self.assertFalse(doc_a == doc_b) - - def test_non_json_dict(self): - self.assertRaises( - errors.InvalidJSON, self.make_document, 'id', 'uid:1', - '"not a json dictionary"') - - def test_non_json(self): - self.assertRaises( - errors.InvalidJSON, self.make_document, 'id', 'uid:1', - 'not a json dictionary') - - def test_get_size(self): - doc_a = self.make_document('a', 'b', '{"some": "content"}') - self.assertEqual( - len('a' + 'b' + '{"some": "content"}'), doc_a.get_size()) - - def test_get_size_empty_document(self): - doc_a = self.make_document('a', 'b', None) - self.assertEqual(len('a' + 'b'), doc_a.get_size()) - - -@skip("Skiping tests imported from U1DB.") -class TestPyDocument(tests.TestCase): - - scenarios = ([( - 'py', {'make_document_for_test': tests.make_document_for_test})]) - - def test_get_content(self): - doc = self.make_document('id', 'rev', '{"content":""}') - self.assertEqual({"content": ""}, doc.content) - doc.set_json('{"content": "new"}') - self.assertEqual({"content": "new"}, doc.content) - - def test_set_content(self): - doc = self.make_document('id', 'rev', '{"content":""}') - doc.content = {"content": "new"} - self.assertEqual('{"content": "new"}', doc.get_json()) - - def test_set_bad_content(self): - doc = self.make_document('id', 'rev', '{"content":""}') - self.assertRaises( - errors.InvalidContent, setattr, doc, 'content', - '{"content": "new"}') - - def test_is_tombstone(self): - doc_a = self.make_document('a', 'b', '{}') - self.assertFalse(doc_a.is_tombstone()) - doc_a.set_json(None) - self.assertTrue(doc_a.is_tombstone()) - - def test_make_tombstone(self): - doc_a = self.make_document('a', 'b', '{}') - self.assertFalse(doc_a.is_tombstone()) - doc_a.make_tombstone() - self.assertTrue(doc_a.is_tombstone()) - - def test_same_content_as(self): - doc_a = self.make_document('a', 'b', '{}') - doc_b = self.make_document('d', 'e', '{}') - self.assertTrue(doc_a.same_content_as(doc_b)) - doc_b = self.make_document('p', 'q', '{}', has_conflicts=True) - self.assertTrue(doc_a.same_content_as(doc_b)) - doc_b.content['key'] = 'value' - self.assertFalse(doc_a.same_content_as(doc_b)) - - def test_same_content_as_json_order(self): - doc_a = self.make_document( - 'a', 'b', '{"key1": "val1", "key2": "val2"}') - doc_b = self.make_document( - 'c', 'd', '{"key2": "val2", "key1": "val1"}') - self.assertTrue(doc_a.same_content_as(doc_b)) - - def test_set_json(self): - doc = self.make_document('id', 'rev', '{"content":""}') - doc.set_json('{"content": "new"}') - self.assertEqual('{"content": "new"}', doc.get_json()) - - def test_set_json_non_dict(self): - doc = self.make_document('id', 'rev', '{"content":""}') - self.assertRaises(errors.InvalidJSON, doc.set_json, '"is not a dict"') - - def test_set_json_error(self): - doc = self.make_document('id', 'rev', '{"content":""}') - self.assertRaises(errors.InvalidJSON, doc.set_json, 'is not json') - - -load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py b/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py deleted file mode 100644 index 344dcb29..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_http_client.py +++ /dev/null @@ -1,360 +0,0 @@ -# Copyright 2011-2012 Canonical Ltd. -# Copyright 2016 LEAP Encryption Access Project -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see . - -""" -Tests for HTTPDatabase -""" -import json - -from unittest import skip - -from leap.soledad.common.l2db import errors -from leap.soledad.common.l2db.remote import http_client - -from leap.soledad.common.tests import u1db_tests as tests - - -@skip("Skiping tests imported from U1DB.") -class TestEncoder(tests.TestCase): - - def test_encode_string(self): - self.assertEqual("foo", http_client._encode_query_parameter("foo")) - - def test_encode_true(self): - self.assertEqual("true", http_client._encode_query_parameter(True)) - - def test_encode_false(self): - self.assertEqual("false", http_client._encode_query_parameter(False)) - - -@skip("Skiping tests imported from U1DB.") -class TestHTTPClientBase(tests.TestCaseWithServer): - - def setUp(self): - super(TestHTTPClientBase, self).setUp() - self.errors = 0 - - def app(self, environ, start_response): - if environ['PATH_INFO'].endswith('echo'): - start_response("200 OK", [('Content-Type', 'application/json')]) - ret = {} - for name in ('REQUEST_METHOD', 'PATH_INFO', 'QUERY_STRING'): - ret[name] = environ[name] - if environ['REQUEST_METHOD'] in ('PUT', 'POST'): - ret['CONTENT_TYPE'] = environ['CONTENT_TYPE'] - content_length = int(environ['CONTENT_LENGTH']) - ret['body'] = environ['wsgi.input'].read(content_length) - return [json.dumps(ret)] - elif environ['PATH_INFO'].endswith('error_then_accept'): - if self.errors >= 3: - start_response( - "200 OK", [('Content-Type', 'application/json')]) - ret = {} - for name in ('REQUEST_METHOD', 'PATH_INFO', 'QUERY_STRING'): - ret[name] = environ[name] - if environ['REQUEST_METHOD'] in ('PUT', 'POST'): - ret['CONTENT_TYPE'] = environ['CONTENT_TYPE'] - content_length = int(environ['CONTENT_LENGTH']) - ret['body'] = '{"oki": "doki"}' - return [json.dumps(ret)] - self.errors += 1 - content_length = int(environ['CONTENT_LENGTH']) - error = json.loads( - environ['wsgi.input'].read(content_length)) - response = error['response'] - # In debug mode, wsgiref has an assertion that the status parameter - # is a 'str' object. However error['status'] returns a unicode - # object. - status = str(error['status']) - if isinstance(response, unicode): - response = str(response) - if isinstance(response, str): - start_response(status, [('Content-Type', 'text/plain')]) - return [str(response)] - else: - start_response(status, [('Content-Type', 'application/json')]) - return [json.dumps(response)] - elif environ['PATH_INFO'].endswith('error'): - self.errors += 1 - content_length = int(environ['CONTENT_LENGTH']) - error = json.loads( - environ['wsgi.input'].read(content_length)) - response = error['response'] - # In debug mode, wsgiref has an assertion that the status parameter - # is a 'str' object. However error['status'] returns a unicode - # object. - status = str(error['status']) - if isinstance(response, unicode): - response = str(response) - if isinstance(response, str): - start_response(status, [('Content-Type', 'text/plain')]) - return [str(response)] - else: - start_response(status, [('Content-Type', 'application/json')]) - return [json.dumps(response)] - elif '/oauth' in environ['PATH_INFO']: - base_url = self.getURL('').rstrip('/') - oauth_req = oauth.OAuthRequest.from_request( - http_method=environ['REQUEST_METHOD'], - http_url=base_url + environ['PATH_INFO'], - headers={'Authorization': environ['HTTP_AUTHORIZATION']}, - query_string=environ['QUERY_STRING'] - ) - oauth_server = oauth.OAuthServer(tests.testingOAuthStore) - oauth_server.add_signature_method(tests.sign_meth_HMAC_SHA1) - try: - consumer, token, params = oauth_server.verify_request( - oauth_req) - except oauth.OAuthError, e: - start_response("401 Unauthorized", - [('Content-Type', 'application/json')]) - return [json.dumps({"error": "unauthorized", - "message": e.message})] - start_response("200 OK", [('Content-Type', 'application/json')]) - return [json.dumps([environ['PATH_INFO'], token.key, params])] - - def make_app(self): - return self.app - - def getClient(self, **kwds): - self.startServer() - return http_client.HTTPClientBase(self.getURL('dbase'), **kwds) - - def test_construct(self): - self.startServer() - url = self.getURL() - cli = http_client.HTTPClientBase(url) - self.assertEqual(url, cli._url.geturl()) - self.assertIs(None, cli._conn) - - def test_parse_url(self): - cli = http_client.HTTPClientBase( - '%s://127.0.0.1:12345/' % self.url_scheme) - self.assertEqual(self.url_scheme, cli._url.scheme) - self.assertEqual('127.0.0.1', cli._url.hostname) - self.assertEqual(12345, cli._url.port) - self.assertEqual('/', cli._url.path) - - def test__ensure_connection(self): - cli = self.getClient() - self.assertIs(None, cli._conn) - cli._ensure_connection() - self.assertIsNot(None, cli._conn) - conn = cli._conn - cli._ensure_connection() - self.assertIs(conn, cli._conn) - - def test_close(self): - cli = self.getClient() - cli._ensure_connection() - cli.close() - self.assertIs(None, cli._conn) - - def test__request(self): - cli = self.getClient() - res, headers = cli._request('PUT', ['echo'], {}, {}) - self.assertEqual({'CONTENT_TYPE': 'application/json', - 'PATH_INFO': '/dbase/echo', - 'QUERY_STRING': '', - 'body': '{}', - 'REQUEST_METHOD': 'PUT'}, json.loads(res)) - - res, headers = cli._request('GET', ['doc', 'echo'], {'a': 1}) - self.assertEqual({'PATH_INFO': '/dbase/doc/echo', - 'QUERY_STRING': 'a=1', - 'REQUEST_METHOD': 'GET'}, json.loads(res)) - - res, headers = cli._request('GET', ['doc', '%FFFF', 'echo'], {'a': 1}) - self.assertEqual({'PATH_INFO': '/dbase/doc/%FFFF/echo', - 'QUERY_STRING': 'a=1', - 'REQUEST_METHOD': 'GET'}, json.loads(res)) - - res, headers = cli._request('POST', ['echo'], {'b': 2}, 'Body', - 'application/x-test') - self.assertEqual({'CONTENT_TYPE': 'application/x-test', - 'PATH_INFO': '/dbase/echo', - 'QUERY_STRING': 'b=2', - 'body': 'Body', - 'REQUEST_METHOD': 'POST'}, json.loads(res)) - - def test__request_json(self): - cli = self.getClient() - res, headers = cli._request_json( - 'POST', ['echo'], {'b': 2}, {'a': 'x'}) - self.assertEqual('application/json', headers['content-type']) - self.assertEqual({'CONTENT_TYPE': 'application/json', - 'PATH_INFO': '/dbase/echo', - 'QUERY_STRING': 'b=2', - 'body': '{"a": "x"}', - 'REQUEST_METHOD': 'POST'}, res) - - def test_unspecified_http_error(self): - cli = self.getClient() - self.assertRaises(errors.HTTPError, - cli._request_json, 'POST', ['error'], {}, - {'status': "500 Internal Error", - 'response': "Crash."}) - try: - cli._request_json('POST', ['error'], {}, - {'status': "500 Internal Error", - 'response': "Fail."}) - except errors.HTTPError, e: - pass - - self.assertEqual(500, e.status) - self.assertEqual("Fail.", e.message) - self.assertTrue("content-type" in e.headers) - - def test_revision_conflict(self): - cli = self.getClient() - self.assertRaises(errors.RevisionConflict, - cli._request_json, 'POST', ['error'], {}, - {'status': "409 Conflict", - 'response': {"error": "revision conflict"}}) - - def test_unavailable_proper(self): - cli = self.getClient() - cli._delays = (0, 0, 0, 0, 0) - self.assertRaises(errors.Unavailable, - cli._request_json, 'POST', ['error'], {}, - {'status': "503 Service Unavailable", - 'response': {"error": "unavailable"}}) - self.assertEqual(5, self.errors) - - def test_unavailable_then_available(self): - cli = self.getClient() - cli._delays = (0, 0, 0, 0, 0) - res, headers = cli._request_json( - 'POST', ['error_then_accept'], {'b': 2}, - {'status': "503 Service Unavailable", - 'response': {"error": "unavailable"}}) - self.assertEqual('application/json', headers['content-type']) - self.assertEqual({'CONTENT_TYPE': 'application/json', - 'PATH_INFO': '/dbase/error_then_accept', - 'QUERY_STRING': 'b=2', - 'body': '{"oki": "doki"}', - 'REQUEST_METHOD': 'POST'}, res) - self.assertEqual(3, self.errors) - - def test_unavailable_random_source(self): - cli = self.getClient() - cli._delays = (0, 0, 0, 0, 0) - try: - cli._request_json('POST', ['error'], {}, - {'status': "503 Service Unavailable", - 'response': "random unavailable."}) - except errors.Unavailable, e: - pass - - self.assertEqual(503, e.status) - self.assertEqual("random unavailable.", e.message) - self.assertTrue("content-type" in e.headers) - self.assertEqual(5, self.errors) - - def test_document_too_big(self): - cli = self.getClient() - self.assertRaises(errors.DocumentTooBig, - cli._request_json, 'POST', ['error'], {}, - {'status': "403 Forbidden", - 'response': {"error": "document too big"}}) - - def test_user_quota_exceeded(self): - cli = self.getClient() - self.assertRaises(errors.UserQuotaExceeded, - cli._request_json, 'POST', ['error'], {}, - {'status': "403 Forbidden", - 'response': {"error": "user quota exceeded"}}) - - def test_user_needs_subscription(self): - cli = self.getClient() - self.assertRaises(errors.SubscriptionNeeded, - cli._request_json, 'POST', ['error'], {}, - {'status': "403 Forbidden", - 'response': {"error": "user needs subscription"}}) - - def test_generic_u1db_error(self): - cli = self.getClient() - self.assertRaises(errors.U1DBError, - cli._request_json, 'POST', ['error'], {}, - {'status': "400 Bad Request", - 'response': {"error": "error"}}) - try: - cli._request_json('POST', ['error'], {}, - {'status': "400 Bad Request", - 'response': {"error": "error"}}) - except errors.U1DBError, e: - pass - self.assertIs(e.__class__, errors.U1DBError) - - def test_unspecified_bad_request(self): - cli = self.getClient() - self.assertRaises(errors.HTTPError, - cli._request_json, 'POST', ['error'], {}, - {'status': "400 Bad Request", - 'response': ""}) - try: - cli._request_json('POST', ['error'], {}, - {'status': "400 Bad Request", - 'response': ""}) - except errors.HTTPError, e: - pass - - self.assertEqual(400, e.status) - self.assertEqual("", e.message) - self.assertTrue("content-type" in e.headers) - - def test_oauth(self): - cli = self.getClient() - cli.set_oauth_credentials(tests.consumer1.key, tests.consumer1.secret, - tests.token1.key, tests.token1.secret) - params = {'x': u'\xf0', 'y': "foo"} - res, headers = cli._request('GET', ['doc', 'oauth'], params) - self.assertEqual( - ['/dbase/doc/oauth', tests.token1.key, params], json.loads(res)) - - # oauth does its own internal quoting - params = {'x': u'\xf0', 'y': "foo"} - res, headers = cli._request('GET', ['doc', 'oauth', 'foo bar'], params) - self.assertEqual( - ['/dbase/doc/oauth/foo bar', tests.token1.key, params], - json.loads(res)) - - def test_oauth_ctr_creds(self): - cli = self.getClient(creds={'oauth': { - 'consumer_key': tests.consumer1.key, - 'consumer_secret': tests.consumer1.secret, - 'token_key': tests.token1.key, - 'token_secret': tests.token1.secret, - }}) - params = {'x': u'\xf0', 'y': "foo"} - res, headers = cli._request('GET', ['doc', 'oauth'], params) - self.assertEqual( - ['/dbase/doc/oauth', tests.token1.key, params], json.loads(res)) - - def test_unknown_creds(self): - self.assertRaises(errors.UnknownAuthMethod, - self.getClient, creds={'foo': {}}) - self.assertRaises(errors.UnknownAuthMethod, - self.getClient, creds={}) - - def test_oauth_Unauthorized(self): - cli = self.getClient() - cli.set_oauth_credentials(tests.consumer1.key, tests.consumer1.secret, - tests.token1.key, "WRONG") - params = {'y': 'foo'} - self.assertRaises(errors.Unauthorized, cli._request, 'GET', - ['doc', 'oauth'], params) diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py b/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py deleted file mode 100644 index 001aebd4..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_http_database.py +++ /dev/null @@ -1,253 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see . - -"""Tests for HTTPDatabase""" - -import inspect -import json - -from unittest import skip - -from leap.soledad.common.l2db import errors -from leap.soledad.common.l2db import Document -from leap.soledad.common.l2db.remote import http_database -from leap.soledad.common.l2db.remote import http_target -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests.u1db_tests import make_http_app - - -@skip("Skiping tests imported from U1DB.") -class TestHTTPDatabaseSimpleOperations(tests.TestCase): - - def setUp(self): - super(TestHTTPDatabaseSimpleOperations, self).setUp() - self.db = http_database.HTTPDatabase('dbase') - self.db._conn = object() # crash if used - self.got = None - self.response_val = None - - def _request(method, url_parts, params=None, body=None, - content_type=None): - self.got = method, url_parts, params, body, content_type - if isinstance(self.response_val, Exception): - raise self.response_val - return self.response_val - - def _request_json(method, url_parts, params=None, body=None, - content_type=None): - self.got = method, url_parts, params, body, content_type - if isinstance(self.response_val, Exception): - raise self.response_val - return self.response_val - - self.db._request = _request - self.db._request_json = _request_json - - def test__sanity_same_signature(self): - my_request_sig = inspect.getargspec(self.db._request) - my_request_sig = (['self'] + my_request_sig[0],) + my_request_sig[1:] - self.assertEqual( - my_request_sig, - inspect.getargspec(http_database.HTTPDatabase._request)) - my_request_json_sig = inspect.getargspec(self.db._request_json) - my_request_json_sig = ((['self'] + my_request_json_sig[0],) + - my_request_json_sig[1:]) - self.assertEqual( - my_request_json_sig, - inspect.getargspec(http_database.HTTPDatabase._request_json)) - - def test__ensure(self): - self.response_val = {'ok': True}, {} - self.db._ensure() - self.assertEqual(('PUT', [], {}, {}, None), self.got) - - def test__delete(self): - self.response_val = {'ok': True}, {} - self.db._delete() - self.assertEqual(('DELETE', [], {}, {}, None), self.got) - - def test__check(self): - self.response_val = {}, {} - res = self.db._check() - self.assertEqual({}, res) - self.assertEqual(('GET', [], None, None, None), self.got) - - def test_put_doc(self): - self.response_val = {'rev': 'doc-rev'}, {} - doc = Document('doc-id', None, '{"v": 1}') - res = self.db.put_doc(doc) - self.assertEqual('doc-rev', res) - self.assertEqual('doc-rev', doc.rev) - self.assertEqual(('PUT', ['doc', 'doc-id'], {}, - '{"v": 1}', 'application/json'), self.got) - - self.response_val = {'rev': 'doc-rev-2'}, {} - doc.content = {"v": 2} - res = self.db.put_doc(doc) - self.assertEqual('doc-rev-2', res) - self.assertEqual('doc-rev-2', doc.rev) - self.assertEqual(('PUT', ['doc', 'doc-id'], {'old_rev': 'doc-rev'}, - '{"v": 2}', 'application/json'), self.got) - - def test_get_doc(self): - self.response_val = '{"v": 2}', {'x-u1db-rev': 'doc-rev', - 'x-u1db-has-conflicts': 'false'} - self.assertGetDoc(self.db, 'doc-id', 'doc-rev', '{"v": 2}', False) - self.assertEqual( - ('GET', ['doc', 'doc-id'], {'include_deleted': False}, None, None), - self.got) - - def test_get_doc_non_existing(self): - self.response_val = errors.DocumentDoesNotExist() - self.assertIs(None, self.db.get_doc('not-there')) - self.assertEqual( - ('GET', ['doc', 'not-there'], {'include_deleted': False}, None, - None), self.got) - - def test_get_doc_deleted(self): - self.response_val = errors.DocumentDoesNotExist() - self.assertIs(None, self.db.get_doc('deleted')) - self.assertEqual( - ('GET', ['doc', 'deleted'], {'include_deleted': False}, None, - None), self.got) - - def test_get_doc_deleted_include_deleted(self): - self.response_val = errors.HTTPError( - 404, - json.dumps({"error": errors.DOCUMENT_DELETED}), - {'x-u1db-rev': 'doc-rev-gone', - 'x-u1db-has-conflicts': 'false'}) - doc = self.db.get_doc('deleted', include_deleted=True) - self.assertEqual('deleted', doc.doc_id) - self.assertEqual('doc-rev-gone', doc.rev) - self.assertIs(None, doc.content) - self.assertEqual( - ('GET', ['doc', 'deleted'], {'include_deleted': True}, None, None), - self.got) - - def test_get_doc_pass_through_errors(self): - self.response_val = errors.HTTPError(500, 'Crash.') - self.assertRaises(errors.HTTPError, - self.db.get_doc, 'something-something') - - def test_create_doc_with_id(self): - self.response_val = {'rev': 'doc-rev'}, {} - new_doc = self.db.create_doc_from_json('{"v": 1}', doc_id='doc-id') - self.assertEqual('doc-rev', new_doc.rev) - self.assertEqual('doc-id', new_doc.doc_id) - self.assertEqual('{"v": 1}', new_doc.get_json()) - self.assertEqual(('PUT', ['doc', 'doc-id'], {}, - '{"v": 1}', 'application/json'), self.got) - - def test_create_doc_without_id(self): - self.response_val = {'rev': 'doc-rev-2'}, {} - new_doc = self.db.create_doc_from_json('{"v": 3}') - self.assertEqual('D-', new_doc.doc_id[:2]) - self.assertEqual('doc-rev-2', new_doc.rev) - self.assertEqual('{"v": 3}', new_doc.get_json()) - self.assertEqual(('PUT', ['doc', new_doc.doc_id], {}, - '{"v": 3}', 'application/json'), self.got) - - def test_delete_doc(self): - self.response_val = {'rev': 'doc-rev-gone'}, {} - doc = Document('doc-id', 'doc-rev', None) - self.db.delete_doc(doc) - self.assertEqual('doc-rev-gone', doc.rev) - self.assertEqual(('DELETE', ['doc', 'doc-id'], {'old_rev': 'doc-rev'}, - None, None), self.got) - - def test_get_sync_target(self): - st = self.db.get_sync_target() - self.assertIsInstance(st, http_target.HTTPSyncTarget) - self.assertEqual(st._url, self.db._url) - - def test_get_sync_target_inherits_oauth_credentials(self): - self.db.set_oauth_credentials(tests.consumer1.key, - tests.consumer1.secret, - tests.token1.key, tests.token1.secret) - st = self.db.get_sync_target() - self.assertEqual(self.db._creds, st._creds) - - -@skip("Skiping tests imported from U1DB.") -class TestHTTPDatabaseCtrWithCreds(tests.TestCase): - - def test_ctr_with_creds(self): - db1 = http_database.HTTPDatabase('http://dbs/db', creds={'oauth': { - 'consumer_key': tests.consumer1.key, - 'consumer_secret': tests.consumer1.secret, - 'token_key': tests.token1.key, - 'token_secret': tests.token1.secret - }}) - self.assertIn('oauth', db1._creds) - - -@skip("Skiping tests imported from U1DB.") -class TestHTTPDatabaseIntegration(tests.TestCaseWithServer): - - make_app_with_state = staticmethod(make_http_app) - - def setUp(self): - super(TestHTTPDatabaseIntegration, self).setUp() - self.startServer() - - def test_non_existing_db(self): - db = http_database.HTTPDatabase(self.getURL('not-there')) - self.assertRaises(errors.DatabaseDoesNotExist, db.get_doc, 'doc1') - - def test__ensure(self): - db = http_database.HTTPDatabase(self.getURL('new')) - db._ensure() - self.assertIs(None, db.get_doc('doc1')) - - def test__delete(self): - self.request_state._create_database('db0') - db = http_database.HTTPDatabase(self.getURL('db0')) - db._delete() - self.assertRaises(errors.DatabaseDoesNotExist, - self.request_state.check_database, 'db0') - - def test_open_database_existing(self): - self.request_state._create_database('db0') - db = http_database.HTTPDatabase.open_database(self.getURL('db0'), - create=False) - self.assertIs(None, db.get_doc('doc1')) - - def test_open_database_non_existing(self): - self.assertRaises(errors.DatabaseDoesNotExist, - http_database.HTTPDatabase.open_database, - self.getURL('not-there'), - create=False) - - def test_open_database_create(self): - db = http_database.HTTPDatabase.open_database(self.getURL('new'), - create=True) - self.assertIs(None, db.get_doc('doc1')) - - def test_delete_database_existing(self): - self.request_state._create_database('db0') - http_database.HTTPDatabase.delete_database(self.getURL('db0')) - self.assertRaises(errors.DatabaseDoesNotExist, - self.request_state.check_database, 'db0') - - def test_doc_ids_needing_quoting(self): - db0 = self.request_state._create_database('db0') - db = http_database.HTTPDatabase.open_database(self.getURL('db0'), - create=False) - doc = Document('%fff', None, '{}') - db.put_doc(doc) - self.assertGetDoc(db0, '%fff', doc.rev, '{}', False) - self.assertGetDoc(db, '%fff', doc.rev, '{}', False) diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_https.py b/common/src/leap/soledad/common/tests/u1db_tests/test_https.py deleted file mode 100644 index 8a5743e7..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_https.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Test support for client-side https support.""" - -import os -import ssl -import sys - -#from paste import httpserver -from unittest import skip - -from leap.soledad.common.l2db.remote import http_client -#from leap.soledad.common.l2db.remote import http_target - -from leap import soledad -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.tests.u1db_tests import make_oauth_http_app - - -#def https_server_def(): - #def make_server(host_port, application): - #from OpenSSL import SSL - #cert_file = os.path.join(os.path.dirname(__file__), 'testing-certs', - #'testing.cert') - #key_file = os.path.join(os.path.dirname(__file__), 'testing-certs', - #'testing.key') - #ssl_context = SSL.Context(SSL.SSLv23_METHOD) - #ssl_context.use_privatekey_file(key_file) - #ssl_context.use_certificate_chain_file(cert_file) - #srv = httpserver.WSGIServerBase(application, host_port, - #httpserver.WSGIHandler, - #ssl_context=ssl_context - #) -# - #def shutdown_request(req): - #req.shutdown() - #srv.close_request(req) -# - #srv.shutdown_request = shutdown_request - #application.base_url = "https://localhost:%s" % srv.server_address[1] - #return srv - #return make_server, "shutdown", "https" - - -#@skip("Skiping tests imported from U1DB.") -#class TestHttpSyncTargetHttpsSupport(tests.TestCaseWithServer): -# - #scenarios = [ - #('oauth_https', {'server_def': https_server_def, - #'make_app_with_state': make_oauth_http_app, - #'make_document_for_test': - #tests.make_document_for_test, - #'sync_target': oauth_https_sync_target - #}), - #] -# - #def setUp(self): - #try: - #import OpenSSL # noqa - #except ImportError: - #self.skipTest("Requires pyOpenSSL") - #self.cacert_pem = os.path.join(os.path.dirname(__file__), - #'testing-certs', 'cacert.pem') - # The default u1db http_client class for doing HTTPS only does HTTPS - # if the platform is linux. Because of this, soledad replaces that - # class with one that will do HTTPS independent of the platform. In - # order to maintain the compatibility with u1db default tests, we undo - # that replacement here. - #http_client._VerifiedHTTPSConnection = \ - #soledad.client.api.old__VerifiedHTTPSConnection - #super(TestHttpSyncTargetHttpsSupport, self).setUp() -# - #def getSyncTarget(self, host, path=None, cert_file=None): - #if self.server is None: - #self.startServer() - #return self.sync_target(self, host, path, cert_file=cert_file) -# - #def test_working(self): - #self.startServer() - #db = self.request_state._create_database('test') - #self.patch(http_client, 'CA_CERTS', self.cacert_pem) - #remote_target = self.getSyncTarget('localhost', 'test') - #remote_target.record_sync_info('other-id', 2, 'T-id') - #self.assertEqual( - #(2, 'T-id'), db._get_replica_gen_and_trans_id('other-id')) -# - #def test_cannot_verify_cert(self): - #if not sys.platform.startswith('linux'): - #self.skipTest( - #"XXX certificate verification happens on linux only for now") - #self.startServer() - # don't print expected traceback server-side - #self.server.handle_error = lambda req, cli_addr: None - #self.request_state._create_database('test') - #remote_target = self.getSyncTarget('localhost', 'test') - #try: - #remote_target.record_sync_info('other-id', 2, 'T-id') - #except ssl.SSLError, e: - #self.assertIn("certificate verify failed", str(e)) - #else: - #self.fail("certificate verification should have failed.") -# - #def test_host_mismatch(self): - #if not sys.platform.startswith('linux'): - #self.skipTest( - #"XXX certificate verification happens on linux only for now") - #self.startServer() - #self.request_state._create_database('test') - #self.patch(http_client, 'CA_CERTS', self.cacert_pem) - #remote_target = self.getSyncTarget('127.0.0.1', 'test') - #self.assertRaises( - #http_client.CertificateError, remote_target.record_sync_info, - #'other-id', 2, 'T-id') -# -# -#load_tests = tests.load_with_scenarios diff --git a/common/src/leap/soledad/common/tests/u1db_tests/test_open.py b/common/src/leap/soledad/common/tests/u1db_tests/test_open.py deleted file mode 100644 index 2fc04e38..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/test_open.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# Copyright 2016 LEAP Encryption Access Project -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see . - -"""Test u1db.open""" - -import os -from unittest import skip - -from leap.soledad.common.l2db import ( - errors, open as u1db_open, -) -from leap.soledad.common.tests import u1db_tests as tests -from leap.soledad.common.l2db.backends import sqlite_backend -from leap.soledad.common.tests.u1db_tests.test_backends \ - import TestAlternativeDocument - - -@skip("Skiping tests imported from U1DB.") -class TestU1DBOpen(tests.TestCase): - - def setUp(self): - super(TestU1DBOpen, self).setUp() - tmpdir = self.createTempDir() - self.db_path = tmpdir + '/test.db' - - def test_open_no_create(self): - self.assertRaises(errors.DatabaseDoesNotExist, - u1db_open, self.db_path, create=False) - self.assertFalse(os.path.exists(self.db_path)) - - def test_open_create(self): - db = u1db_open(self.db_path, create=True) - self.addCleanup(db.close) - self.assertTrue(os.path.exists(self.db_path)) - self.assertIsInstance(db, sqlite_backend.SQLiteDatabase) - - def test_open_with_factory(self): - db = u1db_open(self.db_path, create=True, - document_factory=TestAlternativeDocument) - self.addCleanup(db.close) - self.assertEqual(TestAlternativeDocument, db._factory) - - def test_open_existing(self): - db = sqlite_backend.SQLitePartialExpandDatabase(self.db_path) - self.addCleanup(db.close) - doc = db.create_doc_from_json(tests.simple_doc) - # Even though create=True, we shouldn't wipe the db - db2 = u1db_open(self.db_path, create=True) - self.addCleanup(db2.close) - doc2 = db2.get_doc(doc.doc_id) - self.assertEqual(doc, doc2) - - def test_open_existing_no_create(self): - db = sqlite_backend.SQLitePartialExpandDatabase(self.db_path) - self.addCleanup(db.close) - db2 = u1db_open(self.db_path, create=False) - self.addCleanup(db2.close) - self.assertIsInstance(db2, sqlite_backend.SQLitePartialExpandDatabase) diff --git a/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/Makefile b/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/Makefile deleted file mode 100644 index 2385e75b..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/Makefile +++ /dev/null @@ -1,35 +0,0 @@ -CATOP=./demoCA -ORIG_CONF=/usr/lib/ssl/openssl.cnf -ELEVEN_YEARS=-days 4015 - -init: - cp $(ORIG_CONF) ca.conf - install -d $(CATOP) - install -d $(CATOP)/certs - install -d $(CATOP)/crl - install -d $(CATOP)/newcerts - install -d $(CATOP)/private - touch $(CATOP)/index.txt - echo 01>$(CATOP)/crlnumber - @echo '**** Making CA certificate ...' - openssl req -nodes -new \ - -newkey rsa -keyout $(CATOP)/private/cakey.pem \ - -out $(CATOP)/careq.pem \ - -multivalue-rdn \ - -subj "/C=UK/ST=-/O=u1db LOCAL TESTING ONLY, DO NO TRUST/CN=u1db testing CA" - openssl ca -config ./ca.conf -create_serial \ - -out $(CATOP)/cacert.pem $(ELEVEN_YEARS) -batch \ - -keyfile $(CATOP)/private/cakey.pem -selfsign \ - -extensions v3_ca -infiles $(CATOP)/careq.pem - -pems: - cp ./demoCA/cacert.pem . - openssl req -new -config ca.conf \ - -multivalue-rdn \ - -subj "/O=u1db LOCAL TESTING ONLY, DO NOT TRUST/CN=localhost" \ - -nodes -keyout testing.key -out newreq.pem $(ELEVEN_YEARS) - openssl ca -batch -config ./ca.conf $(ELEVEN_YEARS) \ - -policy policy_anything \ - -out testing.cert -infiles newreq.pem - -.PHONY: init pems diff --git a/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/cacert.pem b/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/cacert.pem deleted file mode 100644 index c019a730..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/cacert.pem +++ /dev/null @@ -1,58 +0,0 @@ -Certificate: - Data: - Version: 3 (0x2) - Serial Number: - e4:de:01:76:c4:78:78:7e - Signature Algorithm: sha1WithRSAEncryption - Issuer: C=UK, ST=-, O=u1db LOCAL TESTING ONLY, DO NO TRUST, CN=u1db testing CA - Validity - Not Before: May 3 11:11:11 2012 GMT - Not After : May 1 11:11:11 2023 GMT - Subject: C=UK, ST=-, O=u1db LOCAL TESTING ONLY, DO NO TRUST, CN=u1db testing CA - Subject Public Key Info: - Public Key Algorithm: rsaEncryption - Public-Key: (1024 bit) - Modulus: - 00:bc:91:a5:7f:7d:37:f7:06:c7:db:5b:83:6a:6b: - 63:c3:8b:5c:f7:84:4d:97:6d:d4:be:bf:e7:79:a8: - c1:03:57:ec:90:d4:20:e7:02:95:d9:a6:49:e3:f9: - 9a:ea:37:b9:b2:02:62:ab:40:d3:42:bb:4a:4e:a2: - 47:71:0f:1d:a2:c5:94:a1:cf:35:d3:23:32:42:c0: - 1e:8d:cb:08:58:fb:8a:5c:3e:ea:eb:d5:2c:ed:d6: - aa:09:b4:b5:7d:e3:45:c9:ae:c2:82:b2:ae:c0:81: - bc:24:06:65:a9:e7:e0:61:ac:25:ee:53:d3:d7:be: - 22:f7:00:a2:ad:c6:0e:3a:39 - Exponent: 65537 (0x10001) - X509v3 extensions: - X509v3 Subject Key Identifier: - DB:3D:93:51:6C:32:15:54:8F:10:50:FC:49:4F:36:15:28:BB:95:6D - X509v3 Authority Key Identifier: - keyid:DB:3D:93:51:6C:32:15:54:8F:10:50:FC:49:4F:36:15:28:BB:95:6D - - X509v3 Basic Constraints: - CA:TRUE - Signature Algorithm: sha1WithRSAEncryption - 72:9b:c1:f7:07:65:83:36:25:4e:01:2f:b7:4a:f2:a4:00:28: - 80:c7:56:2c:32:39:90:13:61:4b:bb:12:c5:44:9d:42:57:85: - 28:19:70:69:e1:43:c8:bd:11:f6:94:df:91:2d:c3:ea:82:8d: - b4:8f:5d:47:a3:00:99:53:29:93:27:6c:c5:da:c1:20:6f:ab: - ec:4a:be:34:f3:8f:02:e5:0c:c0:03:ac:2b:33:41:71:4f:0a: - 72:5a:b4:26:1a:7f:81:bc:c0:95:8a:06:87:a8:11:9f:5c:73: - 38:df:5a:69:40:21:29:ad:46:23:56:75:e1:e9:8b:10:18:4c: - 7b:54 ------BEGIN CERTIFICATE----- -MIICkjCCAfugAwIBAgIJAOTeAXbEeHh+MA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV -BAYTAlVLMQowCAYDVQQIDAEtMS0wKwYDVQQKDCR1MWRiIExPQ0FMIFRFU1RJTkcg -T05MWSwgRE8gTk8gVFJVU1QxGDAWBgNVBAMMD3UxZGIgdGVzdGluZyBDQTAeFw0x -MjA1MDMxMTExMTFaFw0yMzA1MDExMTExMTFaMGIxCzAJBgNVBAYTAlVLMQowCAYD -VQQIDAEtMS0wKwYDVQQKDCR1MWRiIExPQ0FMIFRFU1RJTkcgT05MWSwgRE8gTk8g -VFJVU1QxGDAWBgNVBAMMD3UxZGIgdGVzdGluZyBDQTCBnzANBgkqhkiG9w0BAQEF -AAOBjQAwgYkCgYEAvJGlf3039wbH21uDamtjw4tc94RNl23Uvr/neajBA1fskNQg -5wKV2aZJ4/ma6je5sgJiq0DTQrtKTqJHcQ8dosWUoc810yMyQsAejcsIWPuKXD7q -69Us7daqCbS1feNFya7CgrKuwIG8JAZlqefgYawl7lPT174i9wCircYOOjkCAwEA -AaNQME4wHQYDVR0OBBYEFNs9k1FsMhVUjxBQ/ElPNhUou5VtMB8GA1UdIwQYMBaA -FNs9k1FsMhVUjxBQ/ElPNhUou5VtMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEF -BQADgYEAcpvB9wdlgzYlTgEvt0rypAAogMdWLDI5kBNhS7sSxUSdQleFKBlwaeFD -yL0R9pTfkS3D6oKNtI9dR6MAmVMpkydsxdrBIG+r7Eq+NPOPAuUMwAOsKzNBcU8K -clq0Jhp/gbzAlYoGh6gRn1xzON9aaUAhKa1GI1Z14emLEBhMe1Q= ------END CERTIFICATE----- diff --git a/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/testing.cert b/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/testing.cert deleted file mode 100644 index 985684fb..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/testing.cert +++ /dev/null @@ -1,61 +0,0 @@ -Certificate: - Data: - Version: 3 (0x2) - Serial Number: - e4:de:01:76:c4:78:78:7f - Signature Algorithm: sha1WithRSAEncryption - Issuer: C=UK, ST=-, O=u1db LOCAL TESTING ONLY, DO NO TRUST, CN=u1db testing CA - Validity - Not Before: May 3 11:11:14 2012 GMT - Not After : May 1 11:11:14 2023 GMT - Subject: O=u1db LOCAL TESTING ONLY, DO NOT TRUST, CN=localhost - Subject Public Key Info: - Public Key Algorithm: rsaEncryption - Public-Key: (1024 bit) - Modulus: - 00:c6:1d:72:d3:c5:e4:fc:d1:4c:d9:e4:08:3e:90: - 10:ce:3f:1f:87:4a:1d:4f:7f:2a:5a:52:c9:65:4f: - d9:2c:bf:69:75:18:1a:b5:c9:09:32:00:47:f5:60: - aa:c6:dd:3a:87:37:5f:16:be:de:29:b5:ea:fc:41: - 7e:eb:77:bb:df:63:c3:06:1e:ed:e9:a0:67:1a:f1: - ec:e1:9d:f7:9c:8f:1c:fa:c3:66:7b:39:dc:70:ae: - 09:1b:9c:c0:9a:c4:90:77:45:8e:39:95:a9:2f:92: - 43:bd:27:07:5a:99:51:6e:76:a0:af:dd:b1:2c:8f: - ca:8b:8c:47:0d:f6:6e:fc:69 - Exponent: 65537 (0x10001) - X509v3 extensions: - X509v3 Basic Constraints: - CA:FALSE - Netscape Comment: - OpenSSL Generated Certificate - X509v3 Subject Key Identifier: - 1C:63:85:E1:1D:F3:89:2E:6C:4E:3F:FB:D0:10:64:5A:C1:22:6A:2A - X509v3 Authority Key Identifier: - keyid:DB:3D:93:51:6C:32:15:54:8F:10:50:FC:49:4F:36:15:28:BB:95:6D - - Signature Algorithm: sha1WithRSAEncryption - 1d:6d:3e:bd:93:fd:bd:3e:17:b8:9f:f0:99:7f:db:50:5c:b2: - 01:42:03:b5:d5:94:05:d3:f6:8e:80:82:55:47:1f:58:f2:18: - 6c:ab:ef:43:2c:2f:10:e1:7c:c4:5c:cc:ac:50:50:22:42:aa: - 35:33:f5:b9:f3:a6:66:55:d9:36:f4:f2:e4:d4:d9:b5:2c:52: - 66:d4:21:17:97:22:b8:9b:d7:0e:7c:3d:ce:85:19:ca:c4:d2: - 58:62:31:c6:18:3e:44:fc:f4:30:b6:95:87:ee:21:4a:08:f0: - af:3c:8f:c4:ba:5e:a1:5c:37:1a:7d:7b:fe:66:ae:62:50:17: - 31:ca ------BEGIN CERTIFICATE----- -MIICnzCCAgigAwIBAgIJAOTeAXbEeHh/MA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV -BAYTAlVLMQowCAYDVQQIDAEtMS0wKwYDVQQKDCR1MWRiIExPQ0FMIFRFU1RJTkcg -T05MWSwgRE8gTk8gVFJVU1QxGDAWBgNVBAMMD3UxZGIgdGVzdGluZyBDQTAeFw0x -MjA1MDMxMTExMTRaFw0yMzA1MDExMTExMTRaMEQxLjAsBgNVBAoMJXUxZGIgTE9D -QUwgVEVTVElORyBPTkxZLCBETyBOT1QgVFJVU1QxEjAQBgNVBAMMCWxvY2FsaG9z -dDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxh1y08Xk/NFM2eQIPpAQzj8f -h0odT38qWlLJZU/ZLL9pdRgatckJMgBH9WCqxt06hzdfFr7eKbXq/EF+63e732PD -Bh7t6aBnGvHs4Z33nI8c+sNmeznccK4JG5zAmsSQd0WOOZWpL5JDvScHWplRbnag -r92xLI/Ki4xHDfZu/GkCAwEAAaN7MHkwCQYDVR0TBAIwADAsBglghkgBhvhCAQ0E -HxYdT3BlblNTTCBHZW5lcmF0ZWQgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFBxjheEd -84kubE4/+9AQZFrBImoqMB8GA1UdIwQYMBaAFNs9k1FsMhVUjxBQ/ElPNhUou5Vt -MA0GCSqGSIb3DQEBBQUAA4GBAB1tPr2T/b0+F7if8Jl/21BcsgFCA7XVlAXT9o6A -glVHH1jyGGyr70MsLxDhfMRczKxQUCJCqjUz9bnzpmZV2Tb08uTU2bUsUmbUIReX -Irib1w58Pc6FGcrE0lhiMcYYPkT89DC2lYfuIUoI8K88j8S6XqFcNxp9e/5mrmJQ -FzHK ------END CERTIFICATE----- diff --git a/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/testing.key b/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/testing.key deleted file mode 100644 index d83d4920..00000000 --- a/common/src/leap/soledad/common/tests/u1db_tests/testing-certs/testing.key +++ /dev/null @@ -1,16 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAMYdctPF5PzRTNnk -CD6QEM4/H4dKHU9/KlpSyWVP2Sy/aXUYGrXJCTIAR/VgqsbdOoc3Xxa+3im16vxB -fut3u99jwwYe7emgZxrx7OGd95yPHPrDZns53HCuCRucwJrEkHdFjjmVqS+SQ70n -B1qZUW52oK/dsSyPyouMRw32bvxpAgMBAAECgYBs3lXxhjg1rhabTjIxnx19GTcM -M3Az9V+izweZQu3HJ1CeZiaXauhAr+LbNsniCkRVddotN6oCJdQB10QVxXBZc9Jz -HPJ4zxtZfRZlNMTMmG7eLWrfxpgWnb/BUjDb40yy1nhr9yhDUnI/8RoHDRHnAEHZ -/CnHGUrqcVcrY5zJAQJBAPLhBJg9W88JVmcOKdWxRgs7dLHnZb999Kv1V5mczmAi -jvGvbUmucqOqke6pTUHNYyNHqU6pySzGUi2cH+BAkFECQQDQ0VoAOysg6FVoT15v -tGh57t5sTiCZZ7PS8jwvtThsgA+vcf6c16XWzXgjGXSap4r2QDOY2rI5lsWLaQ8T -+fyZAkAfyFJRmbXp4c7srW3MCOahkaYzoZQu+syJtBFCiMJ40gzik5I5khpuUGPI -V19EvRu8AiSlppIsycb3MPb64XgBAkEAy7DrUf5le5wmc7G4NM6OeyJ+5LbxJbL6 -vnJ8My1a9LuWkVVpQCU7J+UVo2dZTuLPspW9vwTVhUeFOxAoHRxlQQJAFem93f7m -el2BkB2EFqU3onPejkZ5UrDmfmeOQR1axMQNSXqSxcJxqa16Ru1BWV2gcWRbwajQ -oc+kuJThu/r/Ug== ------END PRIVATE KEY----- diff --git a/common/src/leap/soledad/common/tests/util.py b/common/src/leap/soledad/common/tests/util.py deleted file mode 100644 index abe531ce..00000000 --- a/common/src/leap/soledad/common/tests/util.py +++ /dev/null @@ -1,419 +0,0 @@ -# -*- coding: utf-8 -*- -# util.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -""" -Utilities used by multiple test suites. -""" - - -import os -import tempfile -import shutil -import random -import string -import couchdb - -from uuid import uuid4 -from mock import Mock -from urlparse import urljoin -from StringIO import StringIO -from pysqlcipher import dbapi2 - -from twisted.trial import unittest - -from leap.common.testing.basetest import BaseLeapTest - -from leap.soledad.common import l2db -from leap.soledad.common.l2db import sync -from leap.soledad.common.l2db.remote import http_database - -from leap.soledad.common import soledad_assert -from leap.soledad.common.document import SoledadDocument -from leap.soledad.common.couch import CouchDatabase -from leap.soledad.common.couch.state import CouchServerState - -from leap.soledad.common.crypto import ENC_SCHEME_KEY - -from leap.soledad.client import Soledad -from leap.soledad.client import http_target -from leap.soledad.client import auth -from leap.soledad.client.crypto import decrypt_doc_dict -from leap.soledad.client.sqlcipher import SQLCipherDatabase -from leap.soledad.client.sqlcipher import SQLCipherOptions - -from leap.soledad.server import SoledadApp -from leap.soledad.server.auth import SoledadTokenAuthMiddleware - - -PASSWORD = '123456' -ADDRESS = 'leap@leap.se' - - -def make_local_db_and_target(test): - db = test.create_database('test') - st = db.get_sync_target() - return db, st - - -def make_sqlcipher_database_for_test(test, replica_uid): - db = SQLCipherDatabase( - SQLCipherOptions(':memory:', PASSWORD)) - db._set_replica_uid(replica_uid) - return db - - -def copy_sqlcipher_database_for_test(test, db): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS - # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE - # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN - # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR - # HOUSE. - new_db = make_sqlcipher_database_for_test(test, None) - tmpfile = StringIO() - for line in db._db_handle.iterdump(): - if 'sqlite_sequence' not in line: # work around bug in iterdump - tmpfile.write('%s\n' % line) - tmpfile.seek(0) - new_db._db_handle = dbapi2.connect(':memory:') - new_db._db_handle.cursor().executescript(tmpfile.read()) - new_db._db_handle.commit() - new_db._set_replica_uid(db._replica_uid) - new_db._factory = db._factory - return new_db - - -def make_soledad_app(state): - return SoledadApp(state) - - -def make_token_soledad_app(state): - app = SoledadApp(state) - - def _verify_authentication_data(uuid, auth_data): - if uuid.startswith('user-') and auth_data == 'auth-token': - return True - return False - - # we test for action authorization in leap.soledad.common.tests.test_server - def _verify_authorization(uuid, environ): - return True - - application = SoledadTokenAuthMiddleware(app) - application._verify_authentication_data = _verify_authentication_data - application._verify_authorization = _verify_authorization - return application - - -def make_soledad_document_for_test(test, doc_id, rev, content, - has_conflicts=False): - return SoledadDocument( - doc_id, rev, content, has_conflicts=has_conflicts) - - -def make_token_http_database_for_test(test, replica_uid): - test.startServer() - test.request_state._create_database(replica_uid) - - class _HTTPDatabaseWithToken( - http_database.HTTPDatabase, auth.TokenBasedAuth): - - def set_token_credentials(self, uuid, token): - auth.TokenBasedAuth.set_token_credentials(self, uuid, token) - - def _sign_request(self, method, url_query, params): - return auth.TokenBasedAuth._sign_request( - self, method, url_query, params) - - http_db = _HTTPDatabaseWithToken(test.getURL('test')) - http_db.set_token_credentials('user-uuid', 'auth-token') - return http_db - - -def copy_token_http_database_for_test(test, db): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS - # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE - # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN - # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR - # HOUSE. - http_db = test.request_state._copy_database(db) - http_db.set_token_credentials(http_db, 'user-uuid', 'auth-token') - return http_db - - -def sync_via_synchronizer(test, db_source, db_target, trace_hook=None, - trace_hook_shallow=None): - target = db_target.get_sync_target() - trace_hook = trace_hook or trace_hook_shallow - if trace_hook: - target._set_trace_hook(trace_hook) - return sync.Synchronizer(db_source, target).sync() - - -class MockedSharedDBTest(object): - - def get_default_shared_mock(self, put_doc_side_effect=None, - get_doc_return_value=None): - """ - Get a default class for mocking the shared DB - """ - class defaultMockSharedDB(object): - get_doc = Mock(return_value=get_doc_return_value) - put_doc = Mock(side_effect=put_doc_side_effect) - open = Mock(return_value=None) - close = Mock(return_value=None) - syncable = True - - def __call__(self): - return self - return defaultMockSharedDB - - -def soledad_sync_target( - test, path, source_replica_uid=uuid4().hex, - sync_db=None, sync_enc_pool=None): - creds = {'token': { - 'uuid': 'user-uuid', - 'token': 'auth-token', - }} - return http_target.SoledadHTTPSyncTarget( - test.getURL(path), - source_replica_uid, - creds, - test._soledad._crypto, - None, # cert_file - sync_db=sync_db, - sync_enc_pool=sync_enc_pool) - - -# redefine the base leap test class so it inherits from twisted trial's -# TestCase. This is needed so trial knows that it has to manage a reactor and -# wait for deferreds returned by tests to be fired. -BaseLeapTest = type( - 'BaseLeapTest', (unittest.TestCase,), dict(BaseLeapTest.__dict__)) - - -class BaseSoledadTest(BaseLeapTest, MockedSharedDBTest): - - """ - Instantiates Soledad for usage in tests. - """ - defer_sync_encryption = False - - def setUp(self): - # The following snippet comes from BaseLeapTest.setUpClass, but we - # repeat it here because twisted.trial does not work with - # setUpClass/tearDownClass. - - self.old_path = os.environ['PATH'] - self.old_home = os.environ['HOME'] - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.home = self.tempdir - bin_tdir = os.path.join( - self.tempdir, - 'bin') - os.environ["PATH"] = bin_tdir - os.environ["HOME"] = self.tempdir - - # config info - self.db1_file = os.path.join(self.tempdir, "db1.u1db") - self.db2_file = os.path.join(self.tempdir, "db2.u1db") - self.email = ADDRESS - # open test dbs - self._db1 = l2db.open(self.db1_file, create=True, - document_factory=SoledadDocument) - self._db2 = l2db.open(self.db2_file, create=True, - document_factory=SoledadDocument) - # get a random prefix for each test, so we do not mess with - # concurrency during initialization and shutting down of - # each local db. - self.rand_prefix = ''.join( - map(lambda x: random.choice(string.ascii_letters), range(6))) - - # initialize soledad by hand so we can control keys - # XXX check if this soledad is actually used - self._soledad = self._soledad_instance( - prefix=self.rand_prefix, user=self.email) - - def tearDown(self): - self._db1.close() - self._db2.close() - self._soledad.close() - - # restore paths - os.environ["PATH"] = self.old_path - os.environ["HOME"] = self.old_home - - def _delete_temporary_dirs(): - # XXX should not access "private" attrs - for f in [self._soledad.local_db_path, - self._soledad.secrets.secrets_path]: - if os.path.isfile(f): - os.unlink(f) - # The following snippet comes from BaseLeapTest.setUpClass, but we - # repeat it here because twisted.trial does not work with - # setUpClass/tearDownClass. - soledad_assert( - self.tempdir.startswith('/tmp/leap_tests-'), - "beware! tried to remove a dir which does not " - "live in temporal folder!") - shutil.rmtree(self.tempdir) - - from twisted.internet import reactor - reactor.addSystemEventTrigger( - "after", "shutdown", _delete_temporary_dirs) - - def _soledad_instance(self, user=ADDRESS, passphrase=u'123', - prefix='', - secrets_path='secrets.json', - local_db_path='soledad.u1db', - server_url='https://127.0.0.1/', - cert_file=None, - shared_db_class=None, - auth_token='auth-token'): - - def _put_doc_side_effect(doc): - self._doc_put = doc - - if shared_db_class is not None: - MockSharedDB = shared_db_class - else: - MockSharedDB = self.get_default_shared_mock( - _put_doc_side_effect) - - soledad = Soledad( - user, - passphrase, - secrets_path=os.path.join( - self.tempdir, prefix, secrets_path), - local_db_path=os.path.join( - self.tempdir, prefix, local_db_path), - server_url=server_url, # Soledad will fail if not given an url. - cert_file=cert_file, - defer_encryption=self.defer_sync_encryption, - shared_db=MockSharedDB(), - auth_token=auth_token) - self.addCleanup(soledad.close) - return soledad - - def assertGetEncryptedDoc( - self, db, doc_id, doc_rev, content, has_conflicts): - """ - Assert that the document in the database looks correct. - """ - exp_doc = self.make_document(doc_id, doc_rev, content, - has_conflicts=has_conflicts) - doc = db.get_doc(doc_id) - - if ENC_SCHEME_KEY in doc.content: - # XXX check for SYM_KEY too - key = self._soledad._crypto.doc_passphrase(doc.doc_id) - secret = self._soledad._crypto.secret - decrypted = decrypt_doc_dict( - doc.content, doc.doc_id, doc.rev, - key, secret) - doc.set_json(decrypted) - self.assertEqual(exp_doc.doc_id, doc.doc_id) - self.assertEqual(exp_doc.rev, doc.rev) - self.assertEqual(exp_doc.has_conflicts, doc.has_conflicts) - self.assertEqual(exp_doc.content, doc.content) - - -class CouchDBTestCase(unittest.TestCase, MockedSharedDBTest): - - """ - TestCase base class for tests against a real CouchDB server. - """ - - def setUp(self): - """ - Make sure we have a CouchDB instance for a test. - """ - self.couch_port = 5984 - self.couch_url = 'http://localhost:%d' % self.couch_port - self.couch_server = couchdb.Server(self.couch_url) - - def delete_db(self, name): - try: - self.couch_server.delete(name) - except: - # ignore if already missing - pass - - -class CouchServerStateForTests(CouchServerState): - - """ - This is a slightly modified CouchDB server state that allows for creating - a database. - - Ordinarily, the CouchDB server state does not allow some operations, - because for security purposes the Soledad Server should not even have - enough permissions to perform them. For tests, we allow database creation, - otherwise we'd have to create those databases in setUp/tearDown methods, - which is less pleasant than allowing the db to be automatically created. - """ - - def __init__(self, *args, **kwargs): - self.dbs = [] - super(CouchServerStateForTests, self).__init__(*args, **kwargs) - - def _create_database(self, replica_uid=None, dbname=None): - """ - Create db and append to a list, allowing test to close it later - """ - dbname = dbname or ('test-%s' % uuid4().hex) - db = CouchDatabase.open_database( - urljoin(self.couch_url, dbname), - True, - replica_uid=replica_uid or 'test', - ensure_ddocs=True) - self.dbs.append(db) - return db - - def ensure_database(self, dbname): - db = self._create_database(dbname=dbname) - return db, db.replica_uid - - -class SoledadWithCouchServerMixin( - BaseSoledadTest, - CouchDBTestCase): - - def setUp(self): - CouchDBTestCase.setUp(self) - BaseSoledadTest.setUp(self) - main_test_class = getattr(self, 'main_test_class', None) - if main_test_class is not None: - main_test_class.setUp(self) - - def tearDown(self): - main_test_class = getattr(self, 'main_test_class', None) - if main_test_class is not None: - main_test_class.tearDown(self) - # delete the test database - BaseSoledadTest.tearDown(self) - CouchDBTestCase.tearDown(self) - - def make_app(self): - self.request_state = CouchServerStateForTests(self.couch_url) - self.addCleanup(self.delete_dbs) - return self.make_app_with_state(self.request_state) - - def delete_dbs(self): - for db in self.request_state.dbs: - self.delete_db(db._dbname) diff --git a/testing/setup.py b/testing/setup.py new file mode 100644 index 00000000..059b2489 --- /dev/null +++ b/testing/setup.py @@ -0,0 +1,9 @@ +from setuptools import setup +from setuptools import find_packages + + +setup( + name='test_soledad', + packages=find_packages('.'), + package_data={'': ['*.conf']} +) diff --git a/testing/test_soledad/__init__.py b/testing/test_soledad/__init__.py new file mode 100644 index 00000000..c07c8b0e --- /dev/null +++ b/testing/test_soledad/__init__.py @@ -0,0 +1,5 @@ +from test_soledad import util + +__all__ = [ + 'util', +] diff --git a/testing/test_soledad/fixture_soledad.conf b/testing/test_soledad/fixture_soledad.conf new file mode 100644 index 00000000..8d8161c3 --- /dev/null +++ b/testing/test_soledad/fixture_soledad.conf @@ -0,0 +1,11 @@ +[soledad-server] +couch_url = http://soledad:passwd@localhost:5984 +create_cmd = sudo -u soledad-admin /usr/bin/create-user-db +admin_netrc = /etc/couchdb/couchdb-soledad-admin.netrc +batching = 0 + +[database-security] +members = user1, user2 +members_roles = role1, role2 +admins = user3, user4 +admins_roles = role3, role3 diff --git a/testing/test_soledad/u1db_tests/README b/testing/test_soledad/u1db_tests/README new file mode 100644 index 00000000..546dfdc9 --- /dev/null +++ b/testing/test_soledad/u1db_tests/README @@ -0,0 +1,23 @@ +General info +------------ + +Test files in this directory are derived from u1db-0.1.4 tests. The main +difference is that: + + (1) they include the test infrastructure packed with soledad; and + (2) they do not include c_backend_wrapper testing. + +Dependencies +------------ + +u1db tests depend on the following python packages: + + unittest2 + mercurial + hgtools + testtools + discover + testscenarios + paste + routes + cython diff --git a/testing/test_soledad/u1db_tests/__init__.py b/testing/test_soledad/u1db_tests/__init__.py new file mode 100644 index 00000000..ba776864 --- /dev/null +++ b/testing/test_soledad/u1db_tests/__init__.py @@ -0,0 +1,415 @@ +# Copyright 2011-2012 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . +""" +Test infrastructure for U1DB +""" + +import copy +import shutil +import socket +import tempfile +import threading +import json + +from wsgiref import simple_server + +from pysqlcipher import dbapi2 +from StringIO import StringIO + +import testscenarios +from twisted.trial import unittest +from twisted.web.server import Site +from twisted.web.wsgi import WSGIResource +from twisted.internet import reactor + +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db import Document +from leap.soledad.common.l2db.backends import inmemory +from leap.soledad.common.l2db.backends import sqlite_backend +from leap.soledad.common.l2db.remote import server_state +from leap.soledad.common.l2db.remote import http_app +from leap.soledad.common.l2db.remote import http_target + + +class TestCase(unittest.TestCase): + + def createTempDir(self, prefix='u1db-tmp-'): + """Create a temporary directory to do some work in. + + This directory will be scheduled for cleanup when the test ends. + """ + tempdir = tempfile.mkdtemp(prefix=prefix) + self.addCleanup(shutil.rmtree, tempdir) + return tempdir + + def make_document(self, doc_id, doc_rev, content, has_conflicts=False): + return self.make_document_for_test( + self, doc_id, doc_rev, content, has_conflicts) + + def make_document_for_test(self, test, doc_id, doc_rev, content, + has_conflicts): + return make_document_for_test( + test, doc_id, doc_rev, content, has_conflicts) + + def assertGetDoc(self, db, doc_id, doc_rev, content, has_conflicts): + """Assert that the document in the database looks correct.""" + exp_doc = self.make_document(doc_id, doc_rev, content, + has_conflicts=has_conflicts) + self.assertEqual(exp_doc, db.get_doc(doc_id)) + + def assertGetDocIncludeDeleted(self, db, doc_id, doc_rev, content, + has_conflicts): + """Assert that the document in the database looks correct.""" + exp_doc = self.make_document(doc_id, doc_rev, content, + has_conflicts=has_conflicts) + self.assertEqual(exp_doc, db.get_doc(doc_id, include_deleted=True)) + + def assertGetDocConflicts(self, db, doc_id, conflicts): + """Assert what conflicts are stored for a given doc_id. + + :param conflicts: A list of (doc_rev, content) pairs. + The first item must match the first item returned from the + database, however the rest can be returned in any order. + """ + if conflicts: + conflicts = [(rev, + (json.loads(cont) if isinstance(cont, basestring) + else cont)) for (rev, cont) in conflicts] + conflicts = conflicts[:1] + sorted(conflicts[1:]) + actual = db.get_doc_conflicts(doc_id) + if actual: + actual = [ + (doc.rev, (json.loads(doc.get_json()) + if doc.get_json() is not None else None)) + for doc in actual] + actual = actual[:1] + sorted(actual[1:]) + self.assertEqual(conflicts, actual) + + +def multiply_scenarios(a_scenarios, b_scenarios): + """Create the cross-product of scenarios.""" + + all_scenarios = [] + for a_name, a_attrs in a_scenarios: + for b_name, b_attrs in b_scenarios: + name = '%s,%s' % (a_name, b_name) + attrs = dict(a_attrs) + attrs.update(b_attrs) + all_scenarios.append((name, attrs)) + return all_scenarios + + +simple_doc = '{"key": "value"}' +nested_doc = '{"key": "value", "sub": {"doc": "underneath"}}' + + +def make_memory_database_for_test(test, replica_uid): + return inmemory.InMemoryDatabase(replica_uid) + + +def copy_memory_database_for_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR + # HOUSE. + new_db = inmemory.InMemoryDatabase(db._replica_uid) + new_db._transaction_log = db._transaction_log[:] + new_db._docs = copy.deepcopy(db._docs) + new_db._conflicts = copy.deepcopy(db._conflicts) + new_db._indexes = copy.deepcopy(db._indexes) + new_db._factory = db._factory + return new_db + + +def make_sqlite_partial_expanded_for_test(test, replica_uid): + db = sqlite_backend.SQLitePartialExpandDatabase(':memory:') + db._set_replica_uid(replica_uid) + return db + + +def copy_sqlite_partial_expanded_for_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR + # HOUSE. + new_db = sqlite_backend.SQLitePartialExpandDatabase(':memory:') + tmpfile = StringIO() + for line in db._db_handle.iterdump(): + if 'sqlite_sequence' not in line: # work around bug in iterdump + tmpfile.write('%s\n' % line) + tmpfile.seek(0) + new_db._db_handle = dbapi2.connect(':memory:') + new_db._db_handle.cursor().executescript(tmpfile.read()) + new_db._db_handle.commit() + new_db._set_replica_uid(db._replica_uid) + new_db._factory = db._factory + return new_db + + +def make_document_for_test(test, doc_id, rev, content, has_conflicts=False): + return Document(doc_id, rev, content, has_conflicts=has_conflicts) + + +LOCAL_DATABASES_SCENARIOS = [ + ('mem', {'make_database_for_test': make_memory_database_for_test, + 'copy_database_for_test': copy_memory_database_for_test, + 'make_document_for_test': make_document_for_test}), + ('sql', {'make_database_for_test': + make_sqlite_partial_expanded_for_test, + 'copy_database_for_test': + copy_sqlite_partial_expanded_for_test, + 'make_document_for_test': make_document_for_test}), +] + + +class DatabaseBaseTests(TestCase): + + # set to True assertTransactionLog + # is happy with all trans ids = '' + accept_fixed_trans_id = False + + scenarios = LOCAL_DATABASES_SCENARIOS + + def make_database_for_test(self, replica_uid): + return make_memory_database_for_test(self, replica_uid) + + def create_database(self, *args): + return self.make_database_for_test(self, *args) + + def copy_database(self, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES + # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST + # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS + # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND + # NINJA TO YOUR HOUSE. + return self.copy_database_for_test(self, db) + + def setUp(self): + super(DatabaseBaseTests, self).setUp() + self.db = self.create_database('test') + + def tearDown(self): + if hasattr(self, 'db') and self.db is not None: + self.db.close() + super(DatabaseBaseTests, self).tearDown() + + def assertTransactionLog(self, doc_ids, db): + """Assert that the given docs are in the transaction log.""" + log = db._get_transaction_log() + just_ids = [] + seen_transactions = set() + for doc_id, transaction_id in log: + just_ids.append(doc_id) + self.assertIsNot(None, transaction_id, + "Transaction id should not be None") + if transaction_id == '' and self.accept_fixed_trans_id: + continue + self.assertNotEqual('', transaction_id, + "Transaction id should be a unique string") + self.assertTrue(transaction_id.startswith('T-')) + self.assertNotIn(transaction_id, seen_transactions) + seen_transactions.add(transaction_id) + self.assertEqual(doc_ids, just_ids) + + def getLastTransId(self, db): + """Return the transaction id for the last database update.""" + return self.db._get_transaction_log()[-1][-1] + + +class ServerStateForTests(server_state.ServerState): + + """Used in the test suite, so we don't have to touch disk, etc.""" + + def __init__(self): + super(ServerStateForTests, self).__init__() + self._dbs = {} + + def open_database(self, path): + try: + return self._dbs[path] + except KeyError: + raise errors.DatabaseDoesNotExist + + def check_database(self, path): + # cares only about the possible exception + self.open_database(path) + + def ensure_database(self, path): + try: + db = self.open_database(path) + except errors.DatabaseDoesNotExist: + db = self._create_database(path) + return db, db._replica_uid + + def _copy_database(self, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES + # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST + # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS + # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND + # NINJA TO YOUR HOUSE. + new_db = copy_memory_database_for_test(None, db) + path = db._replica_uid + while path in self._dbs: + path += 'copy' + self._dbs[path] = new_db + return new_db + + def _create_database(self, path): + db = inmemory.InMemoryDatabase(path) + self._dbs[path] = db + return db + + def delete_database(self, path): + del self._dbs[path] + + +class ResponderForTests(object): + + """Responder for tests.""" + _started = False + sent_response = False + status = None + + def start_response(self, status='success', **kwargs): + self._started = True + self.status = status + self.kwargs = kwargs + + def send_response(self, status='success', **kwargs): + self.start_response(status, **kwargs) + self.finish_response() + + def finish_response(self): + self.sent_response = True + + +class TestCaseWithServer(TestCase): + + @staticmethod + def server_def(): + # hook point + # should return (ServerClass, "shutdown method name", "url_scheme") + class _RequestHandler(simple_server.WSGIRequestHandler): + + def log_request(*args): + pass # suppress + + def make_server(host_port, application): + assert application, "forgot to override make_app(_with_state)?" + srv = simple_server.WSGIServer(host_port, _RequestHandler) + # patch the value in if it's None + if getattr(application, 'base_url', 1) is None: + application.base_url = "http://%s:%s" % srv.server_address + srv.set_app(application) + return srv + + return make_server, "shutdown", "http" + + @staticmethod + def make_app_with_state(state): + # hook point + return None + + def make_app(self): + # potential hook point + self.request_state = ServerStateForTests() + return self.make_app_with_state(self.request_state) + + def setUp(self): + super(TestCaseWithServer, self).setUp() + self.server = self.server_thread = self.port = None + + def tearDown(self): + if self.server is not None: + self.server.shutdown() + self.server_thread.join() + self.server.server_close() + if self.port: + self.port.stopListening() + super(TestCaseWithServer, self).tearDown() + + @property + def url_scheme(self): + return 'http' + + def startTwistedServer(self): + application = self.make_app() + resource = WSGIResource(reactor, reactor.getThreadPool(), application) + site = Site(resource) + self.port = reactor.listenTCP(0, site, interface='127.0.0.1') + host = self.port.getHost() + self.server_address = (host.host, host.port) + self.addCleanup(self.port.stopListening) + + def startServer(self): + server_def = self.server_def() + server_class, shutdown_meth, _ = server_def + application = self.make_app() + self.server = server_class(('127.0.0.1', 0), application) + self.server_thread = threading.Thread(target=self.server.serve_forever, + kwargs=dict(poll_interval=0.01)) + self.server_thread.start() + self.addCleanup(self.server_thread.join) + self.addCleanup(getattr(self.server, shutdown_meth)) + self.server_address = self.server.server_address + + def getURL(self, path=None): + host, port = self.server_address + if path is None: + path = '' + return '%s://%s:%s/%s' % (self.url_scheme, host, port, path) + + +def socket_pair(): + """Return a pair of TCP sockets connected to each other. + + Unlike socket.socketpair, this should work on Windows. + """ + sock_pair = getattr(socket, 'socket_pair', None) + if sock_pair: + return sock_pair(socket.AF_INET, socket.SOCK_STREAM) + listen_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listen_sock.bind(('127.0.0.1', 0)) + listen_sock.listen(1) + client_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + client_sock.connect(listen_sock.getsockname()) + server_sock, addr = listen_sock.accept() + listen_sock.close() + return server_sock, client_sock + + +def load_with_scenarios(loader, standard_tests, pattern): + """Load the tests in a given module. + + This just applies testscenarios.generate_scenarios to all the tests that + are present. We do it at load time rather than at run time, because it + plays nicer with various tools. + """ + suite = loader.suiteClass() + suite.addTests(testscenarios.generate_scenarios(standard_tests)) + return suite + + +# from u1db.tests.test_remote_sync_target + +def make_http_app(state): + return http_app.HTTPApp(state) + + +def http_sync_target(test, path): + return http_target.HTTPSyncTarget(test.getURL(path)) diff --git a/testing/test_soledad/u1db_tests/test_backends.py b/testing/test_soledad/u1db_tests/test_backends.py new file mode 100644 index 00000000..10dcdff9 --- /dev/null +++ b/testing/test_soledad/u1db_tests/test_backends.py @@ -0,0 +1,1888 @@ +# Copyright 2011 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project +# +# This file is part of leap.soledad.common +# +# leap.soledad.common is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +""" +The backend class for L2DB. This deals with hiding storage details. +""" + +import json + +from leap.soledad.common.l2db import DocumentBase +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db import vectorclock +from leap.soledad.common.l2db.remote import http_database + +from test_soledad import u1db_tests as tests + +from unittest import skip + +simple_doc = tests.simple_doc +nested_doc = tests.nested_doc + + +def make_http_database_for_test(test, replica_uid, path='test', *args): + test.startServer() + test.request_state._create_database(replica_uid) + return http_database.HTTPDatabase(test.getURL(path)) + + +def copy_http_database_for_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR + # HOUSE. + return test.request_state._copy_database(db) + + +class TestAlternativeDocument(DocumentBase): + + """A (not very) alternative implementation of Document.""" + + +@skip("Skiping tests imported from U1DB.") +class AllDatabaseTests(tests.DatabaseBaseTests, tests.TestCaseWithServer): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS + [ + ('http', {'make_database_for_test': make_http_database_for_test, + 'copy_database_for_test': copy_http_database_for_test, + 'make_document_for_test': tests.make_document_for_test, + 'make_app_with_state': tests.make_http_app}), + ] + + def test_close(self): + self.db.close() + + def test_create_doc_allocating_doc_id(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertNotEqual(None, doc.doc_id) + self.assertNotEqual(None, doc.rev) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + + def test_create_doc_different_ids_same_db(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertNotEqual(doc1.doc_id, doc2.doc_id) + + def test_create_doc_with_id(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my-id') + self.assertEqual('my-id', doc.doc_id) + self.assertNotEqual(None, doc.rev) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + + def test_create_doc_existing_id(self): + doc = self.db.create_doc_from_json(simple_doc) + new_content = '{"something": "else"}' + self.assertRaises( + errors.RevisionConflict, self.db.create_doc_from_json, + new_content, doc.doc_id) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + + def test_put_doc_creating_initial(self): + doc = self.make_document('my_doc_id', None, simple_doc) + new_rev = self.db.put_doc(doc) + self.assertIsNot(None, new_rev) + self.assertGetDoc(self.db, 'my_doc_id', new_rev, simple_doc, False) + + def test_put_doc_space_in_id(self): + doc = self.make_document('my doc id', None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + + def test_put_doc_update(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + orig_rev = doc.rev + doc.set_json('{"updated": "stuff"}') + new_rev = self.db.put_doc(doc) + self.assertNotEqual(new_rev, orig_rev) + self.assertGetDoc(self.db, 'my_doc_id', new_rev, + '{"updated": "stuff"}', False) + self.assertEqual(doc.rev, new_rev) + + def test_put_non_ascii_key(self): + content = json.dumps({u'key\xe5': u'val'}) + doc = self.db.create_doc_from_json(content, doc_id='my_doc') + self.assertGetDoc(self.db, 'my_doc', doc.rev, content, False) + + def test_put_non_ascii_value(self): + content = json.dumps({'key': u'\xe5'}) + doc = self.db.create_doc_from_json(content, doc_id='my_doc') + self.assertGetDoc(self.db, 'my_doc', doc.rev, content, False) + + def test_put_doc_refuses_no_id(self): + doc = self.make_document(None, None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + doc = self.make_document("", None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + + def test_put_doc_refuses_slashes(self): + doc = self.make_document('a/b', None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + doc = self.make_document(r'\b', None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + + def test_put_doc_url_quoting_is_fine(self): + doc_id = "%2F%2Ffoo%2Fbar" + doc = self.make_document(doc_id, None, simple_doc) + new_rev = self.db.put_doc(doc) + self.assertGetDoc(self.db, doc_id, new_rev, simple_doc, False) + + def test_put_doc_refuses_non_existing_old_rev(self): + doc = self.make_document('doc-id', 'test:4', simple_doc) + self.assertRaises(errors.RevisionConflict, self.db.put_doc, doc) + + def test_put_doc_refuses_non_ascii_doc_id(self): + doc = self.make_document('d\xc3\xa5c-id', None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + + def test_put_fails_with_bad_old_rev(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + old_rev = doc.rev + bad_doc = self.make_document(doc.doc_id, 'other:1', + '{"something": "else"}') + self.assertRaises(errors.RevisionConflict, self.db.put_doc, bad_doc) + self.assertGetDoc(self.db, 'my_doc_id', old_rev, simple_doc, False) + + def test_create_succeeds_after_delete(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.db.delete_doc(doc) + deleted_doc = self.db.get_doc('my_doc_id', include_deleted=True) + deleted_vc = vectorclock.VectorClockRev(deleted_doc.rev) + new_doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.assertGetDoc(self.db, 'my_doc_id', new_doc.rev, simple_doc, False) + new_vc = vectorclock.VectorClockRev(new_doc.rev) + self.assertTrue( + new_vc.is_newer(deleted_vc), + "%s does not supersede %s" % (new_doc.rev, deleted_doc.rev)) + + def test_put_succeeds_after_delete(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.db.delete_doc(doc) + deleted_doc = self.db.get_doc('my_doc_id', include_deleted=True) + deleted_vc = vectorclock.VectorClockRev(deleted_doc.rev) + doc2 = self.make_document('my_doc_id', None, simple_doc) + self.db.put_doc(doc2) + self.assertGetDoc(self.db, 'my_doc_id', doc2.rev, simple_doc, False) + new_vc = vectorclock.VectorClockRev(doc2.rev) + self.assertTrue( + new_vc.is_newer(deleted_vc), + "%s does not supersede %s" % (doc2.rev, deleted_doc.rev)) + + def test_get_doc_after_put(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.assertGetDoc(self.db, 'my_doc_id', doc.rev, simple_doc, False) + + def test_get_doc_nonexisting(self): + self.assertIs(None, self.db.get_doc('non-existing')) + + def test_get_doc_deleted(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.db.delete_doc(doc) + self.assertIs(None, self.db.get_doc('my_doc_id')) + + def test_get_doc_include_deleted(self): + doc = self.db.create_doc_from_json(simple_doc, doc_id='my_doc_id') + self.db.delete_doc(doc) + self.assertGetDocIncludeDeleted( + self.db, doc.doc_id, doc.rev, None, False) + + def test_get_docs(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertEqual([doc1, doc2], + list(self.db.get_docs([doc1.doc_id, doc2.doc_id]))) + + def test_get_docs_deleted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.db.delete_doc(doc1) + self.assertEqual([doc2], + list(self.db.get_docs([doc1.doc_id, doc2.doc_id]))) + + def test_get_docs_include_deleted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.db.delete_doc(doc1) + self.assertEqual( + [doc1, doc2], + list(self.db.get_docs([doc1.doc_id, doc2.doc_id], + include_deleted=True))) + + def test_get_docs_request_ordered(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertEqual([doc1, doc2], + list(self.db.get_docs([doc1.doc_id, doc2.doc_id]))) + self.assertEqual([doc2, doc1], + list(self.db.get_docs([doc2.doc_id, doc1.doc_id]))) + + def test_get_docs_empty_list(self): + self.assertEqual([], list(self.db.get_docs([]))) + + def test_handles_nested_content(self): + doc = self.db.create_doc_from_json(nested_doc) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, nested_doc, False) + + def test_handles_doc_with_null(self): + doc = self.db.create_doc_from_json('{"key": null}') + self.assertGetDoc(self.db, doc.doc_id, doc.rev, '{"key": null}', False) + + def test_delete_doc(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + orig_rev = doc.rev + self.db.delete_doc(doc) + self.assertNotEqual(orig_rev, doc.rev) + self.assertGetDocIncludeDeleted( + self.db, doc.doc_id, doc.rev, None, False) + self.assertIs(None, self.db.get_doc(doc.doc_id)) + + def test_delete_doc_non_existent(self): + doc = self.make_document('non-existing', 'other:1', simple_doc) + self.assertRaises(errors.DocumentDoesNotExist, self.db.delete_doc, doc) + + def test_delete_doc_already_deleted(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc) + self.assertRaises(errors.DocumentAlreadyDeleted, + self.db.delete_doc, doc) + self.assertGetDocIncludeDeleted( + self.db, doc.doc_id, doc.rev, None, False) + + def test_delete_doc_bad_rev(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) + doc2 = self.make_document(doc1.doc_id, 'other:1', simple_doc) + self.assertRaises(errors.RevisionConflict, self.db.delete_doc, doc2) + self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) + + def test_delete_doc_sets_content_to_None(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc) + self.assertIs(None, doc.get_json()) + + def test_delete_doc_rev_supersedes(self): + doc = self.db.create_doc_from_json(simple_doc) + doc.set_json(nested_doc) + self.db.put_doc(doc) + doc.set_json('{"fishy": "content"}') + self.db.put_doc(doc) + old_rev = doc.rev + self.db.delete_doc(doc) + cur_vc = vectorclock.VectorClockRev(old_rev) + deleted_vc = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(deleted_vc.is_newer(cur_vc), + "%s does not supersede %s" % (doc.rev, old_rev)) + + def test_delete_then_put(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc) + self.assertGetDocIncludeDeleted( + self.db, doc.doc_id, doc.rev, None, False) + doc.set_json(nested_doc) + self.db.put_doc(doc) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, nested_doc, False) + + +@skip("Skiping tests imported from U1DB.") +class DocumentSizeTests(tests.DatabaseBaseTests): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS + + def test_put_doc_refuses_oversized_documents(self): + self.db.set_document_size_limit(1) + doc = self.make_document('doc-id', None, simple_doc) + self.assertRaises(errors.DocumentTooBig, self.db.put_doc, doc) + + def test_create_doc_refuses_oversized_documents(self): + self.db.set_document_size_limit(1) + self.assertRaises( + errors.DocumentTooBig, self.db.create_doc_from_json, simple_doc, + doc_id='my_doc_id') + + def test_set_document_size_limit_zero(self): + self.db.set_document_size_limit(0) + self.assertEqual(0, self.db.document_size_limit) + + def test_set_document_size_limit(self): + self.db.set_document_size_limit(1000000) + self.assertEqual(1000000, self.db.document_size_limit) + + +@skip("Skiping tests imported from U1DB.") +class LocalDatabaseTests(tests.DatabaseBaseTests): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS + + def setUp(self): + tests.DatabaseBaseTests.setUp(self) + + def test_create_doc_different_ids_diff_db(self): + doc1 = self.db.create_doc_from_json(simple_doc) + db2 = self.create_database('other-uid') + doc2 = db2.create_doc_from_json(simple_doc) + self.assertNotEqual(doc1.doc_id, doc2.doc_id) + db2.close() + + def test_put_doc_refuses_slashes_picky(self): + doc = self.make_document('/a', None, simple_doc) + self.assertRaises(errors.InvalidDocId, self.db.put_doc, doc) + + def test_get_all_docs_empty(self): + self.assertEqual([], list(self.db.get_all_docs()[1])) + + def test_get_all_docs(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertEqual( + sorted([doc1, doc2]), sorted(list(self.db.get_all_docs()[1]))) + + def test_get_all_docs_exclude_deleted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.db.delete_doc(doc2) + self.assertEqual([doc1], list(self.db.get_all_docs()[1])) + + def test_get_all_docs_include_deleted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.db.delete_doc(doc2) + self.assertEqual( + sorted([doc1, doc2]), + sorted(list(self.db.get_all_docs(include_deleted=True)[1]))) + + def test_get_all_docs_generation(self): + self.db.create_doc_from_json(simple_doc) + self.db.create_doc_from_json(nested_doc) + self.assertEqual(2, self.db.get_all_docs()[0]) + + def test_simple_put_doc_if_newer(self): + doc = self.make_document('my-doc-id', 'test:1', simple_doc) + state_at_gen = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(('inserted', 1), state_at_gen) + self.assertGetDoc(self.db, 'my-doc-id', 'test:1', simple_doc, False) + + def test_simple_put_doc_if_newer_deleted(self): + self.db.create_doc_from_json('{}', doc_id='my-doc-id') + doc = self.make_document('my-doc-id', 'test:2', None) + state_at_gen = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(('inserted', 2), state_at_gen) + self.assertGetDocIncludeDeleted( + self.db, 'my-doc-id', 'test:2', None, False) + + def test_put_doc_if_newer_already_superseded(self): + orig_doc = '{"new": "doc"}' + doc1 = self.db.create_doc_from_json(orig_doc) + doc1_rev1 = doc1.rev + doc1.set_json(simple_doc) + self.db.put_doc(doc1) + doc1_rev2 = doc1.rev + # Nothing is inserted, because the document is already superseded + doc = self.make_document(doc1.doc_id, doc1_rev1, orig_doc) + state, _ = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual('superseded', state) + self.assertGetDoc(self.db, doc1.doc_id, doc1_rev2, simple_doc, False) + + def test_put_doc_if_newer_autoresolve(self): + doc1 = self.db.create_doc_from_json(simple_doc) + rev = doc1.rev + doc = self.make_document(doc1.doc_id, "whatever:1", doc1.get_json()) + state, _ = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual('superseded', state) + doc2 = self.db.get_doc(doc1.doc_id) + v2 = vectorclock.VectorClockRev(doc2.rev) + self.assertTrue(v2.is_newer(vectorclock.VectorClockRev("whatever:1"))) + self.assertTrue(v2.is_newer(vectorclock.VectorClockRev(rev))) + # strictly newer locally + self.assertTrue(rev not in doc2.rev) + + def test_put_doc_if_newer_already_converged(self): + orig_doc = '{"new": "doc"}' + doc1 = self.db.create_doc_from_json(orig_doc) + state_at_gen = self.db._put_doc_if_newer( + doc1, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(('converged', 1), state_at_gen) + + def test_put_doc_if_newer_conflicted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + # Nothing is inserted, the document id is returned as would-conflict + alt_doc = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + state, _ = self.db._put_doc_if_newer( + alt_doc, save_conflict=False, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual('conflicted', state) + # The database wasn't altered + self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) + + def test_put_doc_if_newer_newer_generation(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + doc = self.make_document('doc_id', 'other:2', simple_doc) + state, _ = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='other', replica_gen=2, + replica_trans_id='T-irrelevant') + self.assertEqual('inserted', state) + + def test_put_doc_if_newer_same_generation_same_txid(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + doc = self.db.create_doc_from_json(simple_doc) + self.make_document(doc.doc_id, 'other:1', simple_doc) + state, _ = self.db._put_doc_if_newer( + doc, save_conflict=False, replica_uid='other', replica_gen=1, + replica_trans_id='T-sid') + self.assertEqual('converged', state) + + def test_put_doc_if_newer_wrong_transaction_id(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + doc = self.make_document('doc_id', 'other:1', simple_doc) + self.assertRaises( + errors.InvalidTransactionId, + self.db._put_doc_if_newer, doc, save_conflict=False, + replica_uid='other', replica_gen=1, replica_trans_id='T-sad') + + def test_put_doc_if_newer_old_generation_older_doc(self): + orig_doc = '{"new": "doc"}' + doc = self.db.create_doc_from_json(orig_doc) + doc_rev1 = doc.rev + doc.set_json(simple_doc) + self.db.put_doc(doc) + self.db._set_replica_gen_and_trans_id('other', 3, 'T-sid') + older_doc = self.make_document(doc.doc_id, doc_rev1, simple_doc) + state, _ = self.db._put_doc_if_newer( + older_doc, save_conflict=False, replica_uid='other', replica_gen=8, + replica_trans_id='T-irrelevant') + self.assertEqual('superseded', state) + + def test_put_doc_if_newer_old_generation_newer_doc(self): + self.db._set_replica_gen_and_trans_id('other', 5, 'T-sid') + doc = self.make_document('doc_id', 'other:1', simple_doc) + self.assertRaises( + errors.InvalidGeneration, + self.db._put_doc_if_newer, doc, save_conflict=False, + replica_uid='other', replica_gen=1, replica_trans_id='T-sad') + + def test_put_doc_if_newer_replica_uid(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + doc2 = self.make_document(doc1.doc_id, doc1.rev + '|other:1', + nested_doc) + self.assertEqual('inserted', + self.db._put_doc_if_newer( + doc2, + save_conflict=False, + replica_uid='other', + replica_gen=2, + replica_trans_id='T-id2')[0]) + self.assertEqual((2, 'T-id2'), self.db._get_replica_gen_and_trans_id( + 'other')) + # Compare to the old rev, should be superseded + doc2 = self.make_document(doc1.doc_id, doc1.rev, nested_doc) + self.assertEqual('superseded', + self.db._put_doc_if_newer( + doc2, + save_conflict=False, + replica_uid='other', + replica_gen=3, + replica_trans_id='T-id3')[0]) + self.assertEqual( + (3, 'T-id3'), self.db._get_replica_gen_and_trans_id('other')) + # A conflict that isn't saved still records the sync gen, because we + # don't need to see it again + doc2 = self.make_document(doc1.doc_id, doc1.rev + '|fourth:1', + '{}') + self.assertEqual('conflicted', + self.db._put_doc_if_newer( + doc2, + save_conflict=False, + replica_uid='other', + replica_gen=4, + replica_trans_id='T-id4')[0]) + self.assertEqual( + (4, 'T-id4'), self.db._get_replica_gen_and_trans_id('other')) + + def test__get_replica_gen_and_trans_id(self): + self.assertEqual( + (0, ''), self.db._get_replica_gen_and_trans_id('other-db')) + self.db._set_replica_gen_and_trans_id('other-db', 2, 'T-transaction') + self.assertEqual( + (2, 'T-transaction'), + self.db._get_replica_gen_and_trans_id('other-db')) + + def test_put_updates_transaction_log(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + doc.set_json('{"something": "else"}') + self.db.put_doc(doc) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual((2, last_trans_id, [(doc.doc_id, 2, last_trans_id)]), + self.db.whats_changed()) + + def test_delete_updates_transaction_log(self): + doc = self.db.create_doc_from_json(simple_doc) + db_gen, _, _ = self.db.whats_changed() + self.db.delete_doc(doc) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual((2, last_trans_id, [(doc.doc_id, 2, last_trans_id)]), + self.db.whats_changed(db_gen)) + + def test_whats_changed_initial_database(self): + self.assertEqual((0, '', []), self.db.whats_changed()) + + def test_whats_changed_returns_one_id_for_multiple_changes(self): + doc = self.db.create_doc_from_json(simple_doc) + doc.set_json('{"new": "contents"}') + self.db.put_doc(doc) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual((2, last_trans_id, [(doc.doc_id, 2, last_trans_id)]), + self.db.whats_changed()) + self.assertEqual((2, last_trans_id, []), self.db.whats_changed(2)) + + def test_whats_changed_returns_last_edits_ascending(self): + doc = self.db.create_doc_from_json(simple_doc) + doc1 = self.db.create_doc_from_json(simple_doc) + doc.set_json('{"new": "contents"}') + self.db.delete_doc(doc1) + delete_trans_id = self.getLastTransId(self.db) + self.db.put_doc(doc) + put_trans_id = self.getLastTransId(self.db) + self.assertEqual((4, put_trans_id, + [(doc1.doc_id, 3, delete_trans_id), + (doc.doc_id, 4, put_trans_id)]), + self.db.whats_changed()) + + def test_whats_changed_doesnt_include_old_gen(self): + self.db.create_doc_from_json(simple_doc) + self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(simple_doc) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual((3, last_trans_id, [(doc2.doc_id, 3, last_trans_id)]), + self.db.whats_changed(2)) + + +@skip("Skiping tests imported from U1DB.") +class LocalDatabaseValidateGenNTransIdTests(tests.DatabaseBaseTests): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS + + def test_validate_gen_and_trans_id(self): + self.db.create_doc_from_json(simple_doc) + gen, trans_id = self.db._get_generation_info() + self.db.validate_gen_and_trans_id(gen, trans_id) + + def test_validate_gen_and_trans_id_invalid_txid(self): + self.db.create_doc_from_json(simple_doc) + gen, _ = self.db._get_generation_info() + self.assertRaises( + errors.InvalidTransactionId, + self.db.validate_gen_and_trans_id, gen, 'wrong') + + def test_validate_gen_and_trans_id_invalid_gen(self): + self.db.create_doc_from_json(simple_doc) + gen, trans_id = self.db._get_generation_info() + self.assertRaises( + errors.InvalidGeneration, + self.db.validate_gen_and_trans_id, gen + 1, trans_id) + + +@skip("Skiping tests imported from U1DB.") +class LocalDatabaseValidateSourceGenTests(tests.DatabaseBaseTests): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS + + def test_validate_source_gen_and_trans_id_same(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + self.db._validate_source('other', 1, 'T-sid') + + def test_validate_source_gen_newer(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + self.db._validate_source('other', 2, 'T-whatevs') + + def test_validate_source_wrong_txid(self): + self.db._set_replica_gen_and_trans_id('other', 1, 'T-sid') + self.assertRaises( + errors.InvalidTransactionId, + self.db._validate_source, 'other', 1, 'T-sad') + + +@skip("Skiping tests imported from U1DB.") +class LocalDatabaseWithConflictsTests(tests.DatabaseBaseTests): + # test supporting/functionality around storing conflicts + + scenarios = tests.LOCAL_DATABASES_SCENARIOS + + def test_get_docs_conflicted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual([doc2], list(self.db.get_docs([doc1.doc_id]))) + + def test_get_docs_conflicts_ignored(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + alt_doc = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + no_conflict_doc = self.make_document(doc1.doc_id, 'alternate:1', + nested_doc) + self.assertEqual([no_conflict_doc, doc2], + list(self.db.get_docs([doc1.doc_id, doc2.doc_id], + check_for_conflicts=False))) + + def test_get_doc_conflicts(self): + doc = self.db.create_doc_from_json(simple_doc) + alt_doc = self.make_document(doc.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual([alt_doc, doc], + self.db.get_doc_conflicts(doc.doc_id)) + + def test_get_all_docs_sees_conflicts(self): + doc = self.db.create_doc_from_json(simple_doc) + alt_doc = self.make_document(doc.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + _, docs = self.db.get_all_docs() + self.assertTrue(list(docs)[0].has_conflicts) + + def test_get_doc_conflicts_unconflicted(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertEqual([], self.db.get_doc_conflicts(doc.doc_id)) + + def test_get_doc_conflicts_no_such_id(self): + self.assertEqual([], self.db.get_doc_conflicts('doc-id')) + + def test_resolve_doc(self): + doc = self.db.create_doc_from_json(simple_doc) + alt_doc = self.make_document(doc.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDocConflicts(self.db, doc.doc_id, + [('alternate:1', nested_doc), + (doc.rev, simple_doc)]) + orig_rev = doc.rev + self.db.resolve_doc(doc, [alt_doc.rev, doc.rev]) + self.assertNotEqual(orig_rev, doc.rev) + self.assertFalse(doc.has_conflicts) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + self.assertGetDocConflicts(self.db, doc.doc_id, []) + + def test_resolve_doc_picks_biggest_vcr(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc2.rev, nested_doc), + (doc1.rev, simple_doc)]) + orig_doc1_rev = doc1.rev + self.db.resolve_doc(doc1, [doc2.rev, doc1.rev]) + self.assertFalse(doc1.has_conflicts) + self.assertNotEqual(orig_doc1_rev, doc1.rev) + self.assertGetDoc(self.db, doc1.doc_id, doc1.rev, simple_doc, False) + self.assertGetDocConflicts(self.db, doc1.doc_id, []) + vcr_1 = vectorclock.VectorClockRev(orig_doc1_rev) + vcr_2 = vectorclock.VectorClockRev(doc2.rev) + vcr_new = vectorclock.VectorClockRev(doc1.rev) + self.assertTrue(vcr_new.is_newer(vcr_1)) + self.assertTrue(vcr_new.is_newer(vcr_2)) + + def test_resolve_doc_partial_not_winning(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc2.rev, nested_doc), + (doc1.rev, simple_doc)]) + content3 = '{"key": "valin3"}' + doc3 = self.make_document(doc1.doc_id, 'third:1', content3) + self.db._put_doc_if_newer( + doc3, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='bar') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc3.rev, content3), + (doc1.rev, simple_doc), + (doc2.rev, nested_doc)]) + self.db.resolve_doc(doc1, [doc2.rev, doc1.rev]) + self.assertTrue(doc1.has_conflicts) + self.assertGetDoc(self.db, doc1.doc_id, doc3.rev, content3, True) + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc3.rev, content3), + (doc1.rev, simple_doc)]) + + def test_resolve_doc_partial_winning(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + content3 = '{"key": "valin3"}' + doc3 = self.make_document(doc1.doc_id, 'third:1', content3) + self.db._put_doc_if_newer( + doc3, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='bar') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc3.rev, content3), + (doc1.rev, simple_doc), + (doc2.rev, nested_doc)]) + self.db.resolve_doc(doc1, [doc3.rev, doc1.rev]) + self.assertTrue(doc1.has_conflicts) + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc1.rev, simple_doc), + (doc2.rev, nested_doc)]) + + def test_resolve_doc_with_delete_conflict(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc1) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc2.rev, nested_doc), + (doc1.rev, None)]) + self.db.resolve_doc(doc2, [doc1.rev, doc2.rev]) + self.assertGetDocConflicts(self.db, doc1.doc_id, []) + self.assertGetDoc(self.db, doc2.doc_id, doc2.rev, nested_doc, False) + + def test_resolve_doc_with_delete_to_delete(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc1) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [(doc2.rev, nested_doc), + (doc1.rev, None)]) + self.db.resolve_doc(doc1, [doc1.rev, doc2.rev]) + self.assertGetDocConflicts(self.db, doc1.doc_id, []) + self.assertGetDocIncludeDeleted( + self.db, doc1.doc_id, doc1.rev, None, False) + + def test_put_doc_if_newer_save_conflicted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + # Document is inserted as a conflict + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + state, _ = self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual('conflicted', state) + # The database was updated + self.assertGetDoc(self.db, doc1.doc_id, doc2.rev, nested_doc, True) + + def test_force_doc_conflict_supersedes_properly(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', '{"b": 1}') + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + doc3 = self.make_document(doc1.doc_id, 'altalt:1', '{"c": 1}') + self.db._put_doc_if_newer( + doc3, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='bar') + doc22 = self.make_document(doc1.doc_id, 'alternate:2', '{"b": 2}') + self.db._put_doc_if_newer( + doc22, save_conflict=True, replica_uid='r', replica_gen=3, + replica_trans_id='zed') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [('alternate:2', doc22.get_json()), + ('altalt:1', doc3.get_json()), + (doc1.rev, simple_doc)]) + + def test_put_doc_if_newer_save_conflict_was_deleted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc1) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertTrue(doc2.has_conflicts) + self.assertGetDoc( + self.db, doc1.doc_id, 'alternate:1', nested_doc, True) + self.assertGetDocConflicts(self.db, doc1.doc_id, + [('alternate:1', nested_doc), + (doc1.rev, None)]) + + def test_put_doc_if_newer_propagates_full_resolution(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + resolved_vcr = vectorclock.VectorClockRev(doc1.rev) + vcr_2 = vectorclock.VectorClockRev(doc2.rev) + resolved_vcr.maximize(vcr_2) + resolved_vcr.increment('alternate') + doc_resolved = self.make_document(doc1.doc_id, resolved_vcr.as_str(), + '{"good": 1}') + state, _ = self.db._put_doc_if_newer( + doc_resolved, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='foo2') + self.assertEqual('inserted', state) + self.assertFalse(doc_resolved.has_conflicts) + self.assertGetDocConflicts(self.db, doc1.doc_id, []) + doc3 = self.db.get_doc(doc1.doc_id) + self.assertFalse(doc3.has_conflicts) + + def test_put_doc_if_newer_propagates_partial_resolution(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'altalt:1', '{}') + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + doc3 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc3, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='foo2') + self.assertGetDocConflicts(self.db, doc1.doc_id, + [('alternate:1', nested_doc), + ('test:1', simple_doc), + ('altalt:1', '{}')]) + resolved_vcr = vectorclock.VectorClockRev(doc1.rev) + vcr_3 = vectorclock.VectorClockRev(doc3.rev) + resolved_vcr.maximize(vcr_3) + resolved_vcr.increment('alternate') + doc_resolved = self.make_document(doc1.doc_id, resolved_vcr.as_str(), + '{"good": 1}') + state, _ = self.db._put_doc_if_newer( + doc_resolved, save_conflict=True, replica_uid='r', replica_gen=3, + replica_trans_id='foo3') + self.assertEqual('inserted', state) + self.assertTrue(doc_resolved.has_conflicts) + doc4 = self.db.get_doc(doc1.doc_id) + self.assertTrue(doc4.has_conflicts) + self.assertGetDocConflicts(self.db, doc1.doc_id, + [('alternate:2|test:1', '{"good": 1}'), + ('altalt:1', '{}')]) + + def test_put_doc_if_newer_replica_uid(self): + doc1 = self.db.create_doc_from_json(simple_doc) + self.db._set_replica_gen_and_trans_id('other', 1, 'T-id') + doc2 = self.make_document(doc1.doc_id, doc1.rev + '|other:1', + nested_doc) + self.db._put_doc_if_newer(doc2, save_conflict=True, + replica_uid='other', replica_gen=2, + replica_trans_id='T-id2') + # Conflict vs the current update + doc2 = self.make_document(doc1.doc_id, doc1.rev + '|third:3', + '{}') + self.assertEqual('conflicted', + self.db._put_doc_if_newer( + doc2, + save_conflict=True, + replica_uid='other', + replica_gen=3, + replica_trans_id='T-id3')[0]) + self.assertEqual( + (3, 'T-id3'), self.db._get_replica_gen_and_trans_id('other')) + + def test_put_doc_if_newer_autoresolve_2(self): + # this is an ordering variant of _3, but that already works + # adding the test explicitly to catch the regression easily + doc_a1 = self.db.create_doc_from_json(simple_doc) + doc_a2 = self.make_document(doc_a1.doc_id, 'test:2', "{}") + doc_a1b1 = self.make_document(doc_a1.doc_id, 'test:1|other:1', + '{"a":"42"}') + doc_a3 = self.make_document(doc_a1.doc_id, 'test:2|other:1', "{}") + state, _ = self.db._put_doc_if_newer( + doc_a2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(state, 'inserted') + state, _ = self.db._put_doc_if_newer( + doc_a1b1, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='foo2') + self.assertEqual(state, 'conflicted') + state, _ = self.db._put_doc_if_newer( + doc_a3, save_conflict=True, replica_uid='r', replica_gen=3, + replica_trans_id='foo3') + self.assertEqual(state, 'inserted') + self.assertFalse(self.db.get_doc(doc_a1.doc_id).has_conflicts) + + def test_put_doc_if_newer_autoresolve_3(self): + doc_a1 = self.db.create_doc_from_json(simple_doc) + doc_a1b1 = self.make_document(doc_a1.doc_id, 'test:1|other:1', "{}") + doc_a2 = self.make_document(doc_a1.doc_id, 'test:2', '{"a":"42"}') + doc_a3 = self.make_document(doc_a1.doc_id, 'test:3', "{}") + state, _ = self.db._put_doc_if_newer( + doc_a1b1, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(state, 'inserted') + state, _ = self.db._put_doc_if_newer( + doc_a2, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='foo2') + self.assertEqual(state, 'conflicted') + state, _ = self.db._put_doc_if_newer( + doc_a3, save_conflict=True, replica_uid='r', replica_gen=3, + replica_trans_id='foo3') + self.assertEqual(state, 'superseded') + doc = self.db.get_doc(doc_a1.doc_id, True) + self.assertFalse(doc.has_conflicts) + rev = vectorclock.VectorClockRev(doc.rev) + rev_a3 = vectorclock.VectorClockRev('test:3') + rev_a1b1 = vectorclock.VectorClockRev('test:1|other:1') + self.assertTrue(rev.is_newer(rev_a3)) + self.assertTrue('test:4' in doc.rev) # locally increased + self.assertTrue(rev.is_newer(rev_a1b1)) + + def test_put_doc_if_newer_autoresolve_4(self): + doc_a1 = self.db.create_doc_from_json(simple_doc) + doc_a1b1 = self.make_document(doc_a1.doc_id, 'test:1|other:1', None) + doc_a2 = self.make_document(doc_a1.doc_id, 'test:2', '{"a":"42"}') + doc_a3 = self.make_document(doc_a1.doc_id, 'test:3', None) + state, _ = self.db._put_doc_if_newer( + doc_a1b1, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertEqual(state, 'inserted') + state, _ = self.db._put_doc_if_newer( + doc_a2, save_conflict=True, replica_uid='r', replica_gen=2, + replica_trans_id='foo2') + self.assertEqual(state, 'conflicted') + state, _ = self.db._put_doc_if_newer( + doc_a3, save_conflict=True, replica_uid='r', replica_gen=3, + replica_trans_id='foo3') + self.assertEqual(state, 'superseded') + doc = self.db.get_doc(doc_a1.doc_id, True) + self.assertFalse(doc.has_conflicts) + rev = vectorclock.VectorClockRev(doc.rev) + rev_a3 = vectorclock.VectorClockRev('test:3') + rev_a1b1 = vectorclock.VectorClockRev('test:1|other:1') + self.assertTrue(rev.is_newer(rev_a3)) + self.assertTrue('test:4' in doc.rev) # locally increased + self.assertTrue(rev.is_newer(rev_a1b1)) + + def test_put_refuses_to_update_conflicted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + content2 = '{"key": "altval"}' + doc2 = self.make_document(doc1.doc_id, 'altrev:1', content2) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDoc(self.db, doc1.doc_id, doc2.rev, content2, True) + content3 = '{"key": "local"}' + doc2.set_json(content3) + self.assertRaises(errors.ConflictedDoc, self.db.put_doc, doc2) + + def test_delete_refuses_for_conflicted(self): + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.make_document(doc1.doc_id, 'altrev:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertGetDoc(self.db, doc2.doc_id, doc2.rev, nested_doc, True) + self.assertRaises(errors.ConflictedDoc, self.db.delete_doc, doc2) + + +@skip("Skiping tests imported from U1DB.") +class DatabaseIndexTests(tests.DatabaseBaseTests): + + scenarios = tests.LOCAL_DATABASES_SCENARIOS + + def assertParseError(self, definition): + self.db.create_doc_from_json(nested_doc) + self.assertRaises( + errors.IndexDefinitionParseError, self.db.create_index, 'idx', + definition) + + def assertIndexCreatable(self, definition): + name = "idx" + self.db.create_doc_from_json(nested_doc) + self.db.create_index(name, definition) + self.assertEqual( + [(name, [definition])], self.db.list_indexes()) + + def test_create_index(self): + self.db.create_index('test-idx', 'name') + self.assertEqual([('test-idx', ['name'])], + self.db.list_indexes()) + + def test_create_index_on_non_ascii_field_name(self): + doc = self.db.create_doc_from_json(json.dumps({u'\xe5': 'value'})) + self.db.create_index('test-idx', u'\xe5') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) + + def test_list_indexes_with_non_ascii_field_names(self): + self.db.create_index('test-idx', u'\xe5') + self.assertEqual( + [('test-idx', [u'\xe5'])], self.db.list_indexes()) + + def test_create_index_evaluates_it(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) + + def test_wildcard_matches_unicode_value(self): + doc = self.db.create_doc_from_json(json.dumps({"key": u"valu\xe5"})) + self.db.create_index('test-idx', 'key') + self.assertEqual([doc], self.db.get_from_index('test-idx', '*')) + + def test_retrieve_unicode_value_from_index(self): + doc = self.db.create_doc_from_json(json.dumps({"key": u"valu\xe5"})) + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc], self.db.get_from_index('test-idx', u"valu\xe5")) + + def test_create_index_fails_if_name_taken(self): + self.db.create_index('test-idx', 'key') + self.assertRaises(errors.IndexNameTakenError, + self.db.create_index, + 'test-idx', 'stuff') + + def test_create_index_does_not_fail_if_name_taken_with_same_index(self): + self.db.create_index('test-idx', 'key') + self.db.create_index('test-idx', 'key') + self.assertEqual([('test-idx', ['key'])], self.db.list_indexes()) + + def test_create_index_does_not_duplicate_indexed_fields(self): + self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.db.delete_index('test-idx') + self.db.create_index('test-idx', 'key') + self.assertEqual(1, len(self.db.get_from_index('test-idx', 'value'))) + + def test_delete_index_does_not_remove_fields_from_other_indexes(self): + self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.db.create_index('test-idx2', 'key') + self.db.delete_index('test-idx') + self.assertEqual(1, len(self.db.get_from_index('test-idx2', 'value'))) + + def test_create_index_after_deleting_document(self): + doc = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc2) + self.db.create_index('test-idx', 'key') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) + + def test_delete_index(self): + self.db.create_index('test-idx', 'key') + self.assertEqual([('test-idx', ['key'])], self.db.list_indexes()) + self.db.delete_index('test-idx') + self.assertEqual([], self.db.list_indexes()) + + def test_create_adds_to_index(self): + self.db.create_index('test-idx', 'key') + doc = self.db.create_doc_from_json(simple_doc) + self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) + + def test_get_from_index_unmatched(self): + self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.assertEqual([], self.db.get_from_index('test-idx', 'novalue')) + + def test_create_index_multiple_exact_matches(self): + doc = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.assertEqual( + sorted([doc, doc2]), + sorted(self.db.get_from_index('test-idx', 'value'))) + + def test_get_from_index(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'value')) + + def test_get_from_index_multi(self): + content = '{"key": "value", "key2": "value2"}' + doc = self.db.create_doc_from_json(content) + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc], self.db.get_from_index('test-idx', 'value', 'value2')) + + def test_get_from_index_multi_list(self): + doc = self.db.create_doc_from_json( + '{"key": "value", "key2": ["value2-1", "value2-2", "value2-3"]}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc], self.db.get_from_index('test-idx', 'value', 'value2-1')) + self.assertEqual( + [doc], self.db.get_from_index('test-idx', 'value', 'value2-2')) + self.assertEqual( + [doc], self.db.get_from_index('test-idx', 'value', 'value2-3')) + self.assertEqual( + [('value', 'value2-1'), ('value', 'value2-2'), + ('value', 'value2-3')], + sorted(self.db.get_index_keys('test-idx'))) + + def test_get_from_index_sees_conflicts(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key', 'key2') + alt_doc = self.make_document( + doc.doc_id, 'alternate:1', + '{"key": "value", "key2": ["value2-1", "value2-2", "value2-3"]}') + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + docs = self.db.get_from_index('test-idx', 'value', 'value2-1') + self.assertTrue(docs[0].has_conflicts) + + def test_get_index_keys_multi_list_list(self): + self.db.create_doc_from_json( + '{"key": "value1-1 value1-2 value1-3", ' + '"key2": ["value2-1", "value2-2", "value2-3"]}') + self.db.create_index('test-idx', 'split_words(key)', 'key2') + self.assertEqual( + [(u'value1-1', u'value2-1'), (u'value1-1', u'value2-2'), + (u'value1-1', u'value2-3'), (u'value1-2', u'value2-1'), + (u'value1-2', u'value2-2'), (u'value1-2', u'value2-3'), + (u'value1-3', u'value2-1'), (u'value1-3', u'value2-2'), + (u'value1-3', u'value2-3')], + sorted(self.db.get_index_keys('test-idx'))) + + def test_get_from_index_multi_ordered(self): + doc1 = self.db.create_doc_from_json( + '{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value3"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value2"}') + doc4 = self.db.create_doc_from_json( + '{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc4, doc3, doc2, doc1], + self.db.get_from_index('test-idx', 'v*', '*')) + + def test_get_range_from_index_start_end(self): + doc1 = self.db.create_doc_from_json('{"key": "value3"}') + doc2 = self.db.create_doc_from_json('{"key": "value2"}') + self.db.create_doc_from_json('{"key": "value4"}') + self.db.create_doc_from_json('{"key": "value1"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc2, doc1], + self.db.get_range_from_index('test-idx', 'value2', 'value3')) + + def test_get_range_from_index_start(self): + doc1 = self.db.create_doc_from_json('{"key": "value3"}') + doc2 = self.db.create_doc_from_json('{"key": "value2"}') + doc3 = self.db.create_doc_from_json('{"key": "value4"}') + self.db.create_doc_from_json('{"key": "value1"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc2, doc1, doc3], + self.db.get_range_from_index('test-idx', 'value2')) + + def test_get_range_from_index_sees_conflicts(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + alt_doc = self.make_document( + doc.doc_id, 'alternate:1', '{"key": "valuedepalue"}') + self.db._put_doc_if_newer( + alt_doc, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + docs = self.db.get_range_from_index('test-idx', 'a') + self.assertTrue(docs[0].has_conflicts) + + def test_get_range_from_index_end(self): + self.db.create_doc_from_json('{"key": "value3"}') + doc2 = self.db.create_doc_from_json('{"key": "value2"}') + self.db.create_doc_from_json('{"key": "value4"}') + doc4 = self.db.create_doc_from_json('{"key": "value1"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc4, doc2], + self.db.get_range_from_index('test-idx', None, 'value2')) + + def test_get_wildcard_range_from_index_start(self): + doc1 = self.db.create_doc_from_json('{"key": "value4"}') + doc2 = self.db.create_doc_from_json('{"key": "value23"}') + doc3 = self.db.create_doc_from_json('{"key": "value2"}') + doc4 = self.db.create_doc_from_json('{"key": "value22"}') + self.db.create_doc_from_json('{"key": "value1"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc3, doc4, doc2, doc1], + self.db.get_range_from_index('test-idx', 'value2*')) + + def test_get_wildcard_range_from_index_end(self): + self.db.create_doc_from_json('{"key": "value4"}') + doc2 = self.db.create_doc_from_json('{"key": "value23"}') + doc3 = self.db.create_doc_from_json('{"key": "value2"}') + doc4 = self.db.create_doc_from_json('{"key": "value22"}') + doc5 = self.db.create_doc_from_json('{"key": "value1"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc5, doc3, doc4, doc2], + self.db.get_range_from_index('test-idx', None, 'value2*')) + + def test_get_wildcard_range_from_index_start_end(self): + self.db.create_doc_from_json('{"key": "a"}') + self.db.create_doc_from_json('{"key": "boo3"}') + doc3 = self.db.create_doc_from_json('{"key": "catalyst"}') + doc4 = self.db.create_doc_from_json('{"key": "whaever"}') + self.db.create_doc_from_json('{"key": "zerg"}') + self.db.create_index('test-idx', 'key') + self.assertEqual( + [doc3, doc4], + self.db.get_range_from_index('test-idx', 'cat*', 'zap*')) + + def test_get_range_from_index_multi_column_start_end(self): + self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value3"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value2"}') + self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc3, doc2], + self.db.get_range_from_index( + 'test-idx', ('value2', 'value2'), ('value2', 'value3'))) + + def test_get_range_from_index_multi_column_start(self): + doc1 = self.db.create_doc_from_json( + '{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value3"}') + self.db.create_doc_from_json('{"key": "value2", "key2": "value2"}') + self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc2, doc1], + self.db.get_range_from_index('test-idx', ('value2', 'value3'))) + + def test_get_range_from_index_multi_column_end(self): + self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value3"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value2"}') + doc4 = self.db.create_doc_from_json( + '{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc4, doc3, doc2], + self.db.get_range_from_index( + 'test-idx', None, ('value2', 'value3'))) + + def test_get_wildcard_range_from_index_multi_column_start(self): + doc1 = self.db.create_doc_from_json( + '{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value23"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value2"}') + self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc3, doc2, doc1], + self.db.get_range_from_index('test-idx', ('value2', 'value2*'))) + + def test_get_wildcard_range_from_index_multi_column_end(self): + self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value23"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value2"}') + doc4 = self.db.create_doc_from_json( + '{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc4, doc3, doc2], + self.db.get_range_from_index( + 'test-idx', None, ('value2', 'value2*'))) + + def test_get_glob_range_from_index_multi_column_start(self): + doc1 = self.db.create_doc_from_json( + '{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value23"}') + self.db.create_doc_from_json('{"key": "value1", "key2": "value2"}') + self.db.create_doc_from_json('{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc2, doc1], + self.db.get_range_from_index('test-idx', ('value2', '*'))) + + def test_get_glob_range_from_index_multi_column_end(self): + self.db.create_doc_from_json('{"key": "value3", "key2": "value4"}') + doc2 = self.db.create_doc_from_json( + '{"key": "value2", "key2": "value23"}') + doc3 = self.db.create_doc_from_json( + '{"key": "value1", "key2": "value2"}') + doc4 = self.db.create_doc_from_json( + '{"key": "value1", "key2": "value1"}') + self.db.create_index('test-idx', 'key', 'key2') + self.assertEqual( + [doc4, doc3, doc2], + self.db.get_range_from_index('test-idx', None, ('value2', '*'))) + + def test_get_range_from_index_illegal_wildcard_order(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_range_from_index, 'test-idx', ('*', 'v2')) + + def test_get_range_from_index_illegal_glob_after_wildcard(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_range_from_index, 'test-idx', ('*', 'v*')) + + def test_get_range_from_index_illegal_wildcard_order_end(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_range_from_index, 'test-idx', None, ('*', 'v2')) + + def test_get_range_from_index_illegal_glob_after_wildcard_end(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_range_from_index, 'test-idx', None, ('*', 'v*')) + + def test_get_from_index_fails_if_no_index(self): + self.assertRaises( + errors.IndexDoesNotExist, self.db.get_from_index, 'foo') + + def test_get_index_keys_fails_if_no_index(self): + self.assertRaises(errors.IndexDoesNotExist, + self.db.get_index_keys, + 'foo') + + def test_get_index_keys_works_if_no_docs(self): + self.db.create_index('test-idx', 'key') + self.assertEqual([], self.db.get_index_keys('test-idx')) + + def test_put_updates_index(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + new_content = '{"key": "altval"}' + doc.set_json(new_content) + self.db.put_doc(doc) + self.assertEqual([], self.db.get_from_index('test-idx', 'value')) + self.assertEqual([doc], self.db.get_from_index('test-idx', 'altval')) + + def test_delete_updates_index(self): + doc = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(simple_doc) + self.db.create_index('test-idx', 'key') + self.assertEqual( + sorted([doc, doc2]), + sorted(self.db.get_from_index('test-idx', 'value'))) + self.db.delete_doc(doc) + self.assertEqual([doc2], self.db.get_from_index('test-idx', 'value')) + + def test_get_from_index_illegal_number_of_entries(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidValueForIndex, self.db.get_from_index, 'test-idx') + self.assertRaises( + errors.InvalidValueForIndex, + self.db.get_from_index, 'test-idx', 'v1') + self.assertRaises( + errors.InvalidValueForIndex, + self.db.get_from_index, 'test-idx', 'v1', 'v2', 'v3') + + def test_get_from_index_illegal_wildcard_order(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_from_index, 'test-idx', '*', 'v2') + + def test_get_from_index_illegal_glob_after_wildcard(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_from_index, 'test-idx', '*', 'v*') + + def test_get_all_from_index(self): + self.db.create_index('test-idx', 'key') + doc1 = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + # This one should not be in the index + self.db.create_doc_from_json('{"no": "key"}') + diff_value_doc = '{"key": "diff value"}' + doc4 = self.db.create_doc_from_json(diff_value_doc) + # This is essentially a 'prefix' match, but we match every entry. + self.assertEqual( + sorted([doc1, doc2, doc4]), + sorted(self.db.get_from_index('test-idx', '*'))) + + def test_get_all_from_index_ordered(self): + self.db.create_index('test-idx', 'key') + doc1 = self.db.create_doc_from_json('{"key": "value x"}') + doc2 = self.db.create_doc_from_json('{"key": "value b"}') + doc3 = self.db.create_doc_from_json('{"key": "value a"}') + doc4 = self.db.create_doc_from_json('{"key": "value m"}') + # This is essentially a 'prefix' match, but we match every entry. + self.assertEqual( + [doc3, doc2, doc4, doc1], self.db.get_from_index('test-idx', '*')) + + def test_put_updates_when_adding_key(self): + doc = self.db.create_doc_from_json("{}") + self.db.create_index('test-idx', 'key') + self.assertEqual([], self.db.get_from_index('test-idx', '*')) + doc.set_json(simple_doc) + self.db.put_doc(doc) + self.assertEqual([doc], self.db.get_from_index('test-idx', '*')) + + def test_get_from_index_empty_string(self): + self.db.create_index('test-idx', 'key') + doc1 = self.db.create_doc_from_json(simple_doc) + content2 = '{"key": ""}' + doc2 = self.db.create_doc_from_json(content2) + self.assertEqual([doc2], self.db.get_from_index('test-idx', '')) + # Empty string matches the wildcard. + self.assertEqual( + sorted([doc1, doc2]), + sorted(self.db.get_from_index('test-idx', '*'))) + + def test_get_from_index_not_null(self): + self.db.create_index('test-idx', 'key') + doc1 = self.db.create_doc_from_json(simple_doc) + self.db.create_doc_from_json('{"key": null}') + self.assertEqual([doc1], self.db.get_from_index('test-idx', '*')) + + def test_get_partial_from_index(self): + content1 = '{"k1": "v1", "k2": "v2"}' + content2 = '{"k1": "v1", "k2": "x2"}' + content3 = '{"k1": "v1", "k2": "y2"}' + # doc4 has a different k1 value, so it doesn't match the prefix. + content4 = '{"k1": "NN", "k2": "v2"}' + doc1 = self.db.create_doc_from_json(content1) + doc2 = self.db.create_doc_from_json(content2) + doc3 = self.db.create_doc_from_json(content3) + self.db.create_doc_from_json(content4) + self.db.create_index('test-idx', 'k1', 'k2') + self.assertEqual( + sorted([doc1, doc2, doc3]), + sorted(self.db.get_from_index('test-idx', "v1", "*"))) + + def test_get_glob_match(self): + # Note: the exact glob syntax is probably subject to change + content1 = '{"k1": "v1", "k2": "v1"}' + content2 = '{"k1": "v1", "k2": "v2"}' + content3 = '{"k1": "v1", "k2": "v3"}' + # doc4 has a different k2 prefix value, so it doesn't match + content4 = '{"k1": "v1", "k2": "ZZ"}' + self.db.create_index('test-idx', 'k1', 'k2') + doc1 = self.db.create_doc_from_json(content1) + doc2 = self.db.create_doc_from_json(content2) + doc3 = self.db.create_doc_from_json(content3) + self.db.create_doc_from_json(content4) + self.assertEqual( + sorted([doc1, doc2, doc3]), + sorted(self.db.get_from_index('test-idx', "v1", "v*"))) + + def test_nested_index(self): + doc = self.db.create_doc_from_json(nested_doc) + self.db.create_index('test-idx', 'sub.doc') + self.assertEqual( + [doc], self.db.get_from_index('test-idx', 'underneath')) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertEqual( + sorted([doc, doc2]), + sorted(self.db.get_from_index('test-idx', 'underneath'))) + + def test_nested_nonexistent(self): + self.db.create_doc_from_json(nested_doc) + # sub exists, but sub.foo does not: + self.db.create_index('test-idx', 'sub.foo') + self.assertEqual([], self.db.get_from_index('test-idx', '*')) + + def test_nested_nonexistent2(self): + self.db.create_doc_from_json(nested_doc) + self.db.create_index('test-idx', 'sub.foo.bar.baz.qux.fnord') + self.assertEqual([], self.db.get_from_index('test-idx', '*')) + + def test_nested_traverses_lists(self): + # subpath finds dicts in list + doc = self.db.create_doc_from_json( + '{"foo": [{"zap": "bar"}, {"zap": "baz"}]}') + # subpath only finds dicts in list + self.db.create_doc_from_json('{"foo": ["zap", "baz"]}') + self.db.create_index('test-idx', 'foo.zap') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'bar')) + self.assertEqual([doc], self.db.get_from_index('test-idx', 'baz')) + + def test_nested_list_traversal(self): + # subpath finds dicts in list + doc = self.db.create_doc_from_json( + '{"foo": [{"zap": [{"qux": "fnord"}, {"qux": "zombo"}]},' + '{"zap": "baz"}]}') + # subpath only finds dicts in list + self.db.create_index('test-idx', 'foo.zap.qux') + self.assertEqual([doc], self.db.get_from_index('test-idx', 'fnord')) + self.assertEqual([doc], self.db.get_from_index('test-idx', 'zombo')) + + def test_index_list1(self): + self.db.create_index("index", "name") + content = '{"name": ["foo", "bar"]}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "bar") + self.assertEqual([doc], rows) + + def test_index_list2(self): + self.db.create_index("index", "name") + content = '{"name": ["foo", "bar"]}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_get_from_index_case_sensitive(self): + self.db.create_index('test-idx', 'key') + doc1 = self.db.create_doc_from_json(simple_doc) + self.assertEqual([], self.db.get_from_index('test-idx', 'V*')) + self.assertEqual([doc1], self.db.get_from_index('test-idx', 'v*')) + + def test_get_from_index_illegal_glob_before_value(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_from_index, 'test-idx', 'v*', 'v2') + + def test_get_from_index_illegal_glob_after_glob(self): + self.db.create_index('test-idx', 'k1', 'k2') + self.assertRaises( + errors.InvalidGlobbing, + self.db.get_from_index, 'test-idx', 'v*', 'v*') + + def test_get_from_index_with_sql_wildcards(self): + self.db.create_index('test-idx', 'key') + content1 = '{"key": "va%lue"}' + content2 = '{"key": "value"}' + content3 = '{"key": "va_lue"}' + doc1 = self.db.create_doc_from_json(content1) + self.db.create_doc_from_json(content2) + doc3 = self.db.create_doc_from_json(content3) + # The '%' in the search should be treated literally, not as a sql + # globbing character. + self.assertEqual([doc1], self.db.get_from_index('test-idx', 'va%*')) + # Same for '_' + self.assertEqual([doc3], self.db.get_from_index('test-idx', 'va_*')) + + def test_get_from_index_with_lower(self): + self.db.create_index("index", "lower(name)") + content = '{"name": "Foo"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_get_from_index_with_lower_matches_same_case(self): + self.db.create_index("index", "lower(name)") + content = '{"name": "foo"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_index_lower_doesnt_match_different_case(self): + self.db.create_index("index", "lower(name)") + content = '{"name": "Foo"}' + self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "Foo") + self.assertEqual([], rows) + + def test_index_lower_doesnt_match_other_index(self): + self.db.create_index("index", "lower(name)") + self.db.create_index("other_index", "name") + content = '{"name": "Foo"}' + self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "Foo") + self.assertEqual(0, len(rows)) + + def test_index_split_words_match_first(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": "foo bar"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_index_split_words_match_second(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": "foo bar"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "bar") + self.assertEqual([doc], rows) + + def test_index_split_words_match_both(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": "foo foo"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_index_split_words_double_space(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": "foo bar"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "bar") + self.assertEqual([doc], rows) + + def test_index_split_words_leading_space(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": " foo bar"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "foo") + self.assertEqual([doc], rows) + + def test_index_split_words_trailing_space(self): + self.db.create_index("index", "split_words(name)") + content = '{"name": "foo bar "}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "bar") + self.assertEqual([doc], rows) + + def test_get_from_index_with_number(self): + self.db.create_index("index", "number(foo, 5)") + content = '{"foo": 12}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "00012") + self.assertEqual([doc], rows) + + def test_get_from_index_with_number_bigger_than_padding(self): + self.db.create_index("index", "number(foo, 5)") + content = '{"foo": 123456}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "123456") + self.assertEqual([doc], rows) + + def test_number_mapping_ignores_non_numbers(self): + self.db.create_index("index", "number(foo, 5)") + content = '{"foo": 56}' + doc1 = self.db.create_doc_from_json(content) + content = '{"foo": "this is not a maigret painting"}' + self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "*") + self.assertEqual([doc1], rows) + + def test_get_from_index_with_bool(self): + self.db.create_index("index", "bool(foo)") + content = '{"foo": true}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "1") + self.assertEqual([doc], rows) + + def test_get_from_index_with_bool_false(self): + self.db.create_index("index", "bool(foo)") + content = '{"foo": false}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "0") + self.assertEqual([doc], rows) + + def test_get_from_index_with_non_bool(self): + self.db.create_index("index", "bool(foo)") + content = '{"foo": 42}' + self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "*") + self.assertEqual([], rows) + + def test_get_from_index_with_combine(self): + self.db.create_index("index", "combine(foo, bar)") + content = '{"foo": "value1", "bar": "value2"}' + doc = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "value1") + self.assertEqual([doc], rows) + rows = self.db.get_from_index("index", "value2") + self.assertEqual([doc], rows) + + def test_get_complex_combine(self): + self.db.create_index( + "index", "combine(number(foo, 5), lower(bar), split_words(baz))") + content = '{"foo": 12, "bar": "ALLCAPS", "baz": "qux nox"}' + doc = self.db.create_doc_from_json(content) + content = '{"foo": "not a number", "bar": "something"}' + doc2 = self.db.create_doc_from_json(content) + rows = self.db.get_from_index("index", "00012") + self.assertEqual([doc], rows) + rows = self.db.get_from_index("index", "allcaps") + self.assertEqual([doc], rows) + rows = self.db.get_from_index("index", "nox") + self.assertEqual([doc], rows) + rows = self.db.get_from_index("index", "something") + self.assertEqual([doc2], rows) + + def test_get_index_keys_from_index(self): + self.db.create_index('test-idx', 'key') + content1 = '{"key": "value1"}' + content2 = '{"key": "value2"}' + content3 = '{"key": "value2"}' + self.db.create_doc_from_json(content1) + self.db.create_doc_from_json(content2) + self.db.create_doc_from_json(content3) + self.assertEqual( + [('value1',), ('value2',)], + sorted(self.db.get_index_keys('test-idx'))) + + def test_get_index_keys_from_multicolumn_index(self): + self.db.create_index('test-idx', 'key1', 'key2') + content1 = '{"key1": "value1", "key2": "val2-1"}' + content2 = '{"key1": "value2", "key2": "val2-2"}' + content3 = '{"key1": "value2", "key2": "val2-2"}' + content4 = '{"key1": "value2", "key2": "val3"}' + self.db.create_doc_from_json(content1) + self.db.create_doc_from_json(content2) + self.db.create_doc_from_json(content3) + self.db.create_doc_from_json(content4) + self.assertEqual([ + ('value1', 'val2-1'), + ('value2', 'val2-2'), + ('value2', 'val3')], + sorted(self.db.get_index_keys('test-idx'))) + + def test_empty_expr(self): + self.assertParseError('') + + def test_nested_unknown_operation(self): + self.assertParseError('unknown_operation(field1)') + + def test_parse_missing_close_paren(self): + self.assertParseError("lower(a") + + def test_parse_trailing_close_paren(self): + self.assertParseError("lower(ab))") + + def test_parse_trailing_chars(self): + self.assertParseError("lower(ab)adsf") + + def test_parse_empty_op(self): + self.assertParseError("(ab)") + + def test_parse_top_level_commas(self): + self.assertParseError("a, b") + + def test_invalid_field_name(self): + self.assertParseError("a.") + + def test_invalid_inner_field_name(self): + self.assertParseError("lower(a.)") + + def test_gobbledigook(self): + self.assertParseError("(@#@cc @#!*DFJSXV(()jccd") + + def test_leading_space(self): + self.assertIndexCreatable(" lower(a)") + + def test_trailing_space(self): + self.assertIndexCreatable("lower(a) ") + + def test_spaces_before_open_paren(self): + self.assertIndexCreatable("lower (a)") + + def test_spaces_after_open_paren(self): + self.assertIndexCreatable("lower( a)") + + def test_spaces_before_close_paren(self): + self.assertIndexCreatable("lower(a )") + + def test_spaces_before_comma(self): + self.assertIndexCreatable("combine(a , b , c)") + + def test_spaces_after_comma(self): + self.assertIndexCreatable("combine(a, b, c)") + + def test_all_together_now(self): + self.assertParseError(' (a) ') + + def test_all_together_now2(self): + self.assertParseError('combine(lower(x)x,foo)') + + +@skip("Skiping tests imported from U1DB.") +class PythonBackendTests(tests.DatabaseBaseTests): + + def setUp(self): + super(PythonBackendTests, self).setUp() + self.simple_doc = json.loads(simple_doc) + + def test_create_doc_with_factory(self): + self.db.set_document_factory(TestAlternativeDocument) + doc = self.db.create_doc(self.simple_doc, doc_id='my_doc_id') + self.assertTrue(isinstance(doc, TestAlternativeDocument)) + + def test_get_doc_after_put_with_factory(self): + doc = self.db.create_doc(self.simple_doc, doc_id='my_doc_id') + self.db.set_document_factory(TestAlternativeDocument) + result = self.db.get_doc('my_doc_id') + self.assertTrue(isinstance(result, TestAlternativeDocument)) + self.assertEqual(doc.doc_id, result.doc_id) + self.assertEqual(doc.rev, result.rev) + self.assertEqual(doc.get_json(), result.get_json()) + self.assertEqual(False, result.has_conflicts) + + def test_get_doc_nonexisting_with_factory(self): + self.db.set_document_factory(TestAlternativeDocument) + self.assertIs(None, self.db.get_doc('non-existing')) + + def test_get_all_docs_with_factory(self): + self.db.set_document_factory(TestAlternativeDocument) + self.db.create_doc(self.simple_doc) + self.assertTrue(isinstance( + list(self.db.get_all_docs()[1])[0], TestAlternativeDocument)) + + def test_get_docs_conflicted_with_factory(self): + self.db.set_document_factory(TestAlternativeDocument) + doc1 = self.db.create_doc(self.simple_doc) + doc2 = self.make_document(doc1.doc_id, 'alternate:1', nested_doc) + self.db._put_doc_if_newer( + doc2, save_conflict=True, replica_uid='r', replica_gen=1, + replica_trans_id='foo') + self.assertTrue( + isinstance( + list(self.db.get_docs([doc1.doc_id]))[0], + TestAlternativeDocument)) + + def test_get_from_index_with_factory(self): + self.db.set_document_factory(TestAlternativeDocument) + self.db.create_doc(self.simple_doc) + self.db.create_index('test-idx', 'key') + self.assertTrue( + isinstance( + self.db.get_from_index('test-idx', 'value')[0], + TestAlternativeDocument)) + + def test_sync_exchange_updates_indexes(self): + doc = self.db.create_doc(self.simple_doc) + self.db.create_index('test-idx', 'key') + new_content = '{"key": "altval"}' + other_rev = 'test:1|z:2' + st = self.db.get_sync_target() + + def ignore(doc_id, doc_rev, doc): + pass + + doc_other = self.make_document(doc.doc_id, other_rev, new_content) + docs_by_gen = [(doc_other, 10, 'T-sid')] + st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=ignore) + self.assertGetDoc(self.db, doc.doc_id, other_rev, new_content, False) + self.assertEqual( + [doc_other], self.db.get_from_index('test-idx', 'altval')) + self.assertEqual([], self.db.get_from_index('test-idx', 'value')) + + +# Use a custom loader to apply the scenarios at load time. +load_tests = tests.load_with_scenarios diff --git a/testing/test_soledad/u1db_tests/test_document.py b/testing/test_soledad/u1db_tests/test_document.py new file mode 100644 index 00000000..a7ead2d1 --- /dev/null +++ b/testing/test_soledad/u1db_tests/test_document.py @@ -0,0 +1,153 @@ +# Copyright 2011 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project +# +# This file is part of leap.soledad.common +# +# leap.soledad.common is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . +from unittest import skip + +from leap.soledad.common.l2db import errors + +from test_soledad import u1db_tests as tests + + +@skip("Skiping tests imported from U1DB.") +class TestDocument(tests.TestCase): + + scenarios = ([( + 'py', {'make_document_for_test': tests.make_document_for_test})]) # + + # tests.C_DATABASE_SCENARIOS) + + def test_create_doc(self): + doc = self.make_document('doc-id', 'uid:1', tests.simple_doc) + self.assertEqual('doc-id', doc.doc_id) + self.assertEqual('uid:1', doc.rev) + self.assertEqual(tests.simple_doc, doc.get_json()) + self.assertFalse(doc.has_conflicts) + + def test__repr__(self): + doc = self.make_document('doc-id', 'uid:1', tests.simple_doc) + self.assertEqual( + '%s(doc-id, uid:1, \'{"key": "value"}\')' + % (doc.__class__.__name__,), + repr(doc)) + + def test__repr__conflicted(self): + doc = self.make_document('doc-id', 'uid:1', tests.simple_doc, + has_conflicts=True) + self.assertEqual( + '%s(doc-id, uid:1, conflicted, \'{"key": "value"}\')' + % (doc.__class__.__name__,), + repr(doc)) + + def test__lt__(self): + doc_a = self.make_document('a', 'b', '{}') + doc_b = self.make_document('b', 'b', '{}') + self.assertTrue(doc_a < doc_b) + self.assertTrue(doc_b > doc_a) + doc_aa = self.make_document('a', 'a', '{}') + self.assertTrue(doc_aa < doc_a) + + def test__eq__(self): + doc_a = self.make_document('a', 'b', '{}') + doc_b = self.make_document('a', 'b', '{}') + self.assertTrue(doc_a == doc_b) + doc_b = self.make_document('a', 'b', '{}', has_conflicts=True) + self.assertFalse(doc_a == doc_b) + + def test_non_json_dict(self): + self.assertRaises( + errors.InvalidJSON, self.make_document, 'id', 'uid:1', + '"not a json dictionary"') + + def test_non_json(self): + self.assertRaises( + errors.InvalidJSON, self.make_document, 'id', 'uid:1', + 'not a json dictionary') + + def test_get_size(self): + doc_a = self.make_document('a', 'b', '{"some": "content"}') + self.assertEqual( + len('a' + 'b' + '{"some": "content"}'), doc_a.get_size()) + + def test_get_size_empty_document(self): + doc_a = self.make_document('a', 'b', None) + self.assertEqual(len('a' + 'b'), doc_a.get_size()) + + +@skip("Skiping tests imported from U1DB.") +class TestPyDocument(tests.TestCase): + + scenarios = ([( + 'py', {'make_document_for_test': tests.make_document_for_test})]) + + def test_get_content(self): + doc = self.make_document('id', 'rev', '{"content":""}') + self.assertEqual({"content": ""}, doc.content) + doc.set_json('{"content": "new"}') + self.assertEqual({"content": "new"}, doc.content) + + def test_set_content(self): + doc = self.make_document('id', 'rev', '{"content":""}') + doc.content = {"content": "new"} + self.assertEqual('{"content": "new"}', doc.get_json()) + + def test_set_bad_content(self): + doc = self.make_document('id', 'rev', '{"content":""}') + self.assertRaises( + errors.InvalidContent, setattr, doc, 'content', + '{"content": "new"}') + + def test_is_tombstone(self): + doc_a = self.make_document('a', 'b', '{}') + self.assertFalse(doc_a.is_tombstone()) + doc_a.set_json(None) + self.assertTrue(doc_a.is_tombstone()) + + def test_make_tombstone(self): + doc_a = self.make_document('a', 'b', '{}') + self.assertFalse(doc_a.is_tombstone()) + doc_a.make_tombstone() + self.assertTrue(doc_a.is_tombstone()) + + def test_same_content_as(self): + doc_a = self.make_document('a', 'b', '{}') + doc_b = self.make_document('d', 'e', '{}') + self.assertTrue(doc_a.same_content_as(doc_b)) + doc_b = self.make_document('p', 'q', '{}', has_conflicts=True) + self.assertTrue(doc_a.same_content_as(doc_b)) + doc_b.content['key'] = 'value' + self.assertFalse(doc_a.same_content_as(doc_b)) + + def test_same_content_as_json_order(self): + doc_a = self.make_document( + 'a', 'b', '{"key1": "val1", "key2": "val2"}') + doc_b = self.make_document( + 'c', 'd', '{"key2": "val2", "key1": "val1"}') + self.assertTrue(doc_a.same_content_as(doc_b)) + + def test_set_json(self): + doc = self.make_document('id', 'rev', '{"content":""}') + doc.set_json('{"content": "new"}') + self.assertEqual('{"content": "new"}', doc.get_json()) + + def test_set_json_non_dict(self): + doc = self.make_document('id', 'rev', '{"content":""}') + self.assertRaises(errors.InvalidJSON, doc.set_json, '"is not a dict"') + + def test_set_json_error(self): + doc = self.make_document('id', 'rev', '{"content":""}') + self.assertRaises(errors.InvalidJSON, doc.set_json, 'is not json') + + +load_tests = tests.load_with_scenarios diff --git a/testing/test_soledad/u1db_tests/test_http_client.py b/testing/test_soledad/u1db_tests/test_http_client.py new file mode 100644 index 00000000..e9516236 --- /dev/null +++ b/testing/test_soledad/u1db_tests/test_http_client.py @@ -0,0 +1,304 @@ +# Copyright 2011-2012 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +""" +Tests for HTTPDatabase +""" +import json + +from unittest import skip + +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db.remote import http_client + +from test_soledad import u1db_tests as tests + + +@skip("Skiping tests imported from U1DB.") +class TestEncoder(tests.TestCase): + + def test_encode_string(self): + self.assertEqual("foo", http_client._encode_query_parameter("foo")) + + def test_encode_true(self): + self.assertEqual("true", http_client._encode_query_parameter(True)) + + def test_encode_false(self): + self.assertEqual("false", http_client._encode_query_parameter(False)) + + +@skip("Skiping tests imported from U1DB.") +class TestHTTPClientBase(tests.TestCaseWithServer): + + def setUp(self): + super(TestHTTPClientBase, self).setUp() + self.errors = 0 + + def app(self, environ, start_response): + if environ['PATH_INFO'].endswith('echo'): + start_response("200 OK", [('Content-Type', 'application/json')]) + ret = {} + for name in ('REQUEST_METHOD', 'PATH_INFO', 'QUERY_STRING'): + ret[name] = environ[name] + if environ['REQUEST_METHOD'] in ('PUT', 'POST'): + ret['CONTENT_TYPE'] = environ['CONTENT_TYPE'] + content_length = int(environ['CONTENT_LENGTH']) + ret['body'] = environ['wsgi.input'].read(content_length) + return [json.dumps(ret)] + elif environ['PATH_INFO'].endswith('error_then_accept'): + if self.errors >= 3: + start_response( + "200 OK", [('Content-Type', 'application/json')]) + ret = {} + for name in ('REQUEST_METHOD', 'PATH_INFO', 'QUERY_STRING'): + ret[name] = environ[name] + if environ['REQUEST_METHOD'] in ('PUT', 'POST'): + ret['CONTENT_TYPE'] = environ['CONTENT_TYPE'] + content_length = int(environ['CONTENT_LENGTH']) + ret['body'] = '{"oki": "doki"}' + return [json.dumps(ret)] + self.errors += 1 + content_length = int(environ['CONTENT_LENGTH']) + error = json.loads( + environ['wsgi.input'].read(content_length)) + response = error['response'] + # In debug mode, wsgiref has an assertion that the status parameter + # is a 'str' object. However error['status'] returns a unicode + # object. + status = str(error['status']) + if isinstance(response, unicode): + response = str(response) + if isinstance(response, str): + start_response(status, [('Content-Type', 'text/plain')]) + return [str(response)] + else: + start_response(status, [('Content-Type', 'application/json')]) + return [json.dumps(response)] + elif environ['PATH_INFO'].endswith('error'): + self.errors += 1 + content_length = int(environ['CONTENT_LENGTH']) + error = json.loads( + environ['wsgi.input'].read(content_length)) + response = error['response'] + # In debug mode, wsgiref has an assertion that the status parameter + # is a 'str' object. However error['status'] returns a unicode + # object. + status = str(error['status']) + if isinstance(response, unicode): + response = str(response) + if isinstance(response, str): + start_response(status, [('Content-Type', 'text/plain')]) + return [str(response)] + else: + start_response(status, [('Content-Type', 'application/json')]) + return [json.dumps(response)] + + def make_app(self): + return self.app + + def getClient(self, **kwds): + self.startServer() + return http_client.HTTPClientBase(self.getURL('dbase'), **kwds) + + def test_construct(self): + self.startServer() + url = self.getURL() + cli = http_client.HTTPClientBase(url) + self.assertEqual(url, cli._url.geturl()) + self.assertIs(None, cli._conn) + + def test_parse_url(self): + cli = http_client.HTTPClientBase( + '%s://127.0.0.1:12345/' % self.url_scheme) + self.assertEqual(self.url_scheme, cli._url.scheme) + self.assertEqual('127.0.0.1', cli._url.hostname) + self.assertEqual(12345, cli._url.port) + self.assertEqual('/', cli._url.path) + + def test__ensure_connection(self): + cli = self.getClient() + self.assertIs(None, cli._conn) + cli._ensure_connection() + self.assertIsNot(None, cli._conn) + conn = cli._conn + cli._ensure_connection() + self.assertIs(conn, cli._conn) + + def test_close(self): + cli = self.getClient() + cli._ensure_connection() + cli.close() + self.assertIs(None, cli._conn) + + def test__request(self): + cli = self.getClient() + res, headers = cli._request('PUT', ['echo'], {}, {}) + self.assertEqual({'CONTENT_TYPE': 'application/json', + 'PATH_INFO': '/dbase/echo', + 'QUERY_STRING': '', + 'body': '{}', + 'REQUEST_METHOD': 'PUT'}, json.loads(res)) + + res, headers = cli._request('GET', ['doc', 'echo'], {'a': 1}) + self.assertEqual({'PATH_INFO': '/dbase/doc/echo', + 'QUERY_STRING': 'a=1', + 'REQUEST_METHOD': 'GET'}, json.loads(res)) + + res, headers = cli._request('GET', ['doc', '%FFFF', 'echo'], {'a': 1}) + self.assertEqual({'PATH_INFO': '/dbase/doc/%FFFF/echo', + 'QUERY_STRING': 'a=1', + 'REQUEST_METHOD': 'GET'}, json.loads(res)) + + res, headers = cli._request('POST', ['echo'], {'b': 2}, 'Body', + 'application/x-test') + self.assertEqual({'CONTENT_TYPE': 'application/x-test', + 'PATH_INFO': '/dbase/echo', + 'QUERY_STRING': 'b=2', + 'body': 'Body', + 'REQUEST_METHOD': 'POST'}, json.loads(res)) + + def test__request_json(self): + cli = self.getClient() + res, headers = cli._request_json( + 'POST', ['echo'], {'b': 2}, {'a': 'x'}) + self.assertEqual('application/json', headers['content-type']) + self.assertEqual({'CONTENT_TYPE': 'application/json', + 'PATH_INFO': '/dbase/echo', + 'QUERY_STRING': 'b=2', + 'body': '{"a": "x"}', + 'REQUEST_METHOD': 'POST'}, res) + + def test_unspecified_http_error(self): + cli = self.getClient() + self.assertRaises(errors.HTTPError, + cli._request_json, 'POST', ['error'], {}, + {'status': "500 Internal Error", + 'response': "Crash."}) + try: + cli._request_json('POST', ['error'], {}, + {'status': "500 Internal Error", + 'response': "Fail."}) + except errors.HTTPError, e: + pass + + self.assertEqual(500, e.status) + self.assertEqual("Fail.", e.message) + self.assertTrue("content-type" in e.headers) + + def test_revision_conflict(self): + cli = self.getClient() + self.assertRaises(errors.RevisionConflict, + cli._request_json, 'POST', ['error'], {}, + {'status': "409 Conflict", + 'response': {"error": "revision conflict"}}) + + def test_unavailable_proper(self): + cli = self.getClient() + cli._delays = (0, 0, 0, 0, 0) + self.assertRaises(errors.Unavailable, + cli._request_json, 'POST', ['error'], {}, + {'status': "503 Service Unavailable", + 'response': {"error": "unavailable"}}) + self.assertEqual(5, self.errors) + + def test_unavailable_then_available(self): + cli = self.getClient() + cli._delays = (0, 0, 0, 0, 0) + res, headers = cli._request_json( + 'POST', ['error_then_accept'], {'b': 2}, + {'status': "503 Service Unavailable", + 'response': {"error": "unavailable"}}) + self.assertEqual('application/json', headers['content-type']) + self.assertEqual({'CONTENT_TYPE': 'application/json', + 'PATH_INFO': '/dbase/error_then_accept', + 'QUERY_STRING': 'b=2', + 'body': '{"oki": "doki"}', + 'REQUEST_METHOD': 'POST'}, res) + self.assertEqual(3, self.errors) + + def test_unavailable_random_source(self): + cli = self.getClient() + cli._delays = (0, 0, 0, 0, 0) + try: + cli._request_json('POST', ['error'], {}, + {'status': "503 Service Unavailable", + 'response': "random unavailable."}) + except errors.Unavailable, e: + pass + + self.assertEqual(503, e.status) + self.assertEqual("random unavailable.", e.message) + self.assertTrue("content-type" in e.headers) + self.assertEqual(5, self.errors) + + def test_document_too_big(self): + cli = self.getClient() + self.assertRaises(errors.DocumentTooBig, + cli._request_json, 'POST', ['error'], {}, + {'status': "403 Forbidden", + 'response': {"error": "document too big"}}) + + def test_user_quota_exceeded(self): + cli = self.getClient() + self.assertRaises(errors.UserQuotaExceeded, + cli._request_json, 'POST', ['error'], {}, + {'status': "403 Forbidden", + 'response': {"error": "user quota exceeded"}}) + + def test_user_needs_subscription(self): + cli = self.getClient() + self.assertRaises(errors.SubscriptionNeeded, + cli._request_json, 'POST', ['error'], {}, + {'status': "403 Forbidden", + 'response': {"error": "user needs subscription"}}) + + def test_generic_u1db_error(self): + cli = self.getClient() + self.assertRaises(errors.U1DBError, + cli._request_json, 'POST', ['error'], {}, + {'status': "400 Bad Request", + 'response': {"error": "error"}}) + try: + cli._request_json('POST', ['error'], {}, + {'status': "400 Bad Request", + 'response': {"error": "error"}}) + except errors.U1DBError, e: + pass + self.assertIs(e.__class__, errors.U1DBError) + + def test_unspecified_bad_request(self): + cli = self.getClient() + self.assertRaises(errors.HTTPError, + cli._request_json, 'POST', ['error'], {}, + {'status': "400 Bad Request", + 'response': ""}) + try: + cli._request_json('POST', ['error'], {}, + {'status': "400 Bad Request", + 'response': ""}) + except errors.HTTPError, e: + pass + + self.assertEqual(400, e.status) + self.assertEqual("", e.message) + self.assertTrue("content-type" in e.headers) + + def test_unknown_creds(self): + self.assertRaises(errors.UnknownAuthMethod, + self.getClient, creds={'foo': {}}) + self.assertRaises(errors.UnknownAuthMethod, + self.getClient, creds={}) diff --git a/testing/test_soledad/u1db_tests/test_http_database.py b/testing/test_soledad/u1db_tests/test_http_database.py new file mode 100644 index 00000000..a3ed9361 --- /dev/null +++ b/testing/test_soledad/u1db_tests/test_http_database.py @@ -0,0 +1,233 @@ +# Copyright 2011 Canonical Ltd. +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +"""Tests for HTTPDatabase""" + +import inspect +import json + +from unittest import skip + +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db import Document +from leap.soledad.common.l2db.remote import http_database +from leap.soledad.common.l2db.remote import http_target +from test_soledad import u1db_tests as tests +from test_soledad.u1db_tests import make_http_app + + +@skip("Skiping tests imported from U1DB.") +class TestHTTPDatabaseSimpleOperations(tests.TestCase): + + def setUp(self): + super(TestHTTPDatabaseSimpleOperations, self).setUp() + self.db = http_database.HTTPDatabase('dbase') + self.db._conn = object() # crash if used + self.got = None + self.response_val = None + + def _request(method, url_parts, params=None, body=None, + content_type=None): + self.got = method, url_parts, params, body, content_type + if isinstance(self.response_val, Exception): + raise self.response_val + return self.response_val + + def _request_json(method, url_parts, params=None, body=None, + content_type=None): + self.got = method, url_parts, params, body, content_type + if isinstance(self.response_val, Exception): + raise self.response_val + return self.response_val + + self.db._request = _request + self.db._request_json = _request_json + + def test__sanity_same_signature(self): + my_request_sig = inspect.getargspec(self.db._request) + my_request_sig = (['self'] + my_request_sig[0],) + my_request_sig[1:] + self.assertEqual( + my_request_sig, + inspect.getargspec(http_database.HTTPDatabase._request)) + my_request_json_sig = inspect.getargspec(self.db._request_json) + my_request_json_sig = ((['self'] + my_request_json_sig[0],) + + my_request_json_sig[1:]) + self.assertEqual( + my_request_json_sig, + inspect.getargspec(http_database.HTTPDatabase._request_json)) + + def test__ensure(self): + self.response_val = {'ok': True}, {} + self.db._ensure() + self.assertEqual(('PUT', [], {}, {}, None), self.got) + + def test__delete(self): + self.response_val = {'ok': True}, {} + self.db._delete() + self.assertEqual(('DELETE', [], {}, {}, None), self.got) + + def test__check(self): + self.response_val = {}, {} + res = self.db._check() + self.assertEqual({}, res) + self.assertEqual(('GET', [], None, None, None), self.got) + + def test_put_doc(self): + self.response_val = {'rev': 'doc-rev'}, {} + doc = Document('doc-id', None, '{"v": 1}') + res = self.db.put_doc(doc) + self.assertEqual('doc-rev', res) + self.assertEqual('doc-rev', doc.rev) + self.assertEqual(('PUT', ['doc', 'doc-id'], {}, + '{"v": 1}', 'application/json'), self.got) + + self.response_val = {'rev': 'doc-rev-2'}, {} + doc.content = {"v": 2} + res = self.db.put_doc(doc) + self.assertEqual('doc-rev-2', res) + self.assertEqual('doc-rev-2', doc.rev) + self.assertEqual(('PUT', ['doc', 'doc-id'], {'old_rev': 'doc-rev'}, + '{"v": 2}', 'application/json'), self.got) + + def test_get_doc(self): + self.response_val = '{"v": 2}', {'x-u1db-rev': 'doc-rev', + 'x-u1db-has-conflicts': 'false'} + self.assertGetDoc(self.db, 'doc-id', 'doc-rev', '{"v": 2}', False) + self.assertEqual( + ('GET', ['doc', 'doc-id'], {'include_deleted': False}, None, None), + self.got) + + def test_get_doc_non_existing(self): + self.response_val = errors.DocumentDoesNotExist() + self.assertIs(None, self.db.get_doc('not-there')) + self.assertEqual( + ('GET', ['doc', 'not-there'], {'include_deleted': False}, None, + None), self.got) + + def test_get_doc_deleted(self): + self.response_val = errors.DocumentDoesNotExist() + self.assertIs(None, self.db.get_doc('deleted')) + self.assertEqual( + ('GET', ['doc', 'deleted'], {'include_deleted': False}, None, + None), self.got) + + def test_get_doc_deleted_include_deleted(self): + self.response_val = errors.HTTPError( + 404, + json.dumps({"error": errors.DOCUMENT_DELETED}), + {'x-u1db-rev': 'doc-rev-gone', + 'x-u1db-has-conflicts': 'false'}) + doc = self.db.get_doc('deleted', include_deleted=True) + self.assertEqual('deleted', doc.doc_id) + self.assertEqual('doc-rev-gone', doc.rev) + self.assertIs(None, doc.content) + self.assertEqual( + ('GET', ['doc', 'deleted'], {'include_deleted': True}, None, None), + self.got) + + def test_get_doc_pass_through_errors(self): + self.response_val = errors.HTTPError(500, 'Crash.') + self.assertRaises(errors.HTTPError, + self.db.get_doc, 'something-something') + + def test_create_doc_with_id(self): + self.response_val = {'rev': 'doc-rev'}, {} + new_doc = self.db.create_doc_from_json('{"v": 1}', doc_id='doc-id') + self.assertEqual('doc-rev', new_doc.rev) + self.assertEqual('doc-id', new_doc.doc_id) + self.assertEqual('{"v": 1}', new_doc.get_json()) + self.assertEqual(('PUT', ['doc', 'doc-id'], {}, + '{"v": 1}', 'application/json'), self.got) + + def test_create_doc_without_id(self): + self.response_val = {'rev': 'doc-rev-2'}, {} + new_doc = self.db.create_doc_from_json('{"v": 3}') + self.assertEqual('D-', new_doc.doc_id[:2]) + self.assertEqual('doc-rev-2', new_doc.rev) + self.assertEqual('{"v": 3}', new_doc.get_json()) + self.assertEqual(('PUT', ['doc', new_doc.doc_id], {}, + '{"v": 3}', 'application/json'), self.got) + + def test_delete_doc(self): + self.response_val = {'rev': 'doc-rev-gone'}, {} + doc = Document('doc-id', 'doc-rev', None) + self.db.delete_doc(doc) + self.assertEqual('doc-rev-gone', doc.rev) + self.assertEqual(('DELETE', ['doc', 'doc-id'], {'old_rev': 'doc-rev'}, + None, None), self.got) + + def test_get_sync_target(self): + st = self.db.get_sync_target() + self.assertIsInstance(st, http_target.HTTPSyncTarget) + self.assertEqual(st._url, self.db._url) + + +@skip("Skiping tests imported from U1DB.") +class TestHTTPDatabaseIntegration(tests.TestCaseWithServer): + + make_app_with_state = staticmethod(make_http_app) + + def setUp(self): + super(TestHTTPDatabaseIntegration, self).setUp() + self.startServer() + + def test_non_existing_db(self): + db = http_database.HTTPDatabase(self.getURL('not-there')) + self.assertRaises(errors.DatabaseDoesNotExist, db.get_doc, 'doc1') + + def test__ensure(self): + db = http_database.HTTPDatabase(self.getURL('new')) + db._ensure() + self.assertIs(None, db.get_doc('doc1')) + + def test__delete(self): + self.request_state._create_database('db0') + db = http_database.HTTPDatabase(self.getURL('db0')) + db._delete() + self.assertRaises(errors.DatabaseDoesNotExist, + self.request_state.check_database, 'db0') + + def test_open_database_existing(self): + self.request_state._create_database('db0') + db = http_database.HTTPDatabase.open_database(self.getURL('db0'), + create=False) + self.assertIs(None, db.get_doc('doc1')) + + def test_open_database_non_existing(self): + self.assertRaises(errors.DatabaseDoesNotExist, + http_database.HTTPDatabase.open_database, + self.getURL('not-there'), + create=False) + + def test_open_database_create(self): + db = http_database.HTTPDatabase.open_database(self.getURL('new'), + create=True) + self.assertIs(None, db.get_doc('doc1')) + + def test_delete_database_existing(self): + self.request_state._create_database('db0') + http_database.HTTPDatabase.delete_database(self.getURL('db0')) + self.assertRaises(errors.DatabaseDoesNotExist, + self.request_state.check_database, 'db0') + + def test_doc_ids_needing_quoting(self): + db0 = self.request_state._create_database('db0') + db = http_database.HTTPDatabase.open_database(self.getURL('db0'), + create=False) + doc = Document('%fff', None, '{}') + db.put_doc(doc) + self.assertGetDoc(db0, '%fff', doc.rev, '{}', False) + self.assertGetDoc(db, '%fff', doc.rev, '{}', False) diff --git a/testing/test_soledad/u1db_tests/test_https.py b/testing/test_soledad/u1db_tests/test_https.py new file mode 100644 index 00000000..baffa723 --- /dev/null +++ b/testing/test_soledad/u1db_tests/test_https.py @@ -0,0 +1,105 @@ +"""Test support for client-side https support.""" + +import os +import ssl +import sys + +from paste import httpserver +from unittest import skip + +from leap.soledad.common.l2db.remote import http_client + +from leap import soledad +from test_soledad import u1db_tests as tests + + +def https_server_def(): + def make_server(host_port, application): + from OpenSSL import SSL + cert_file = os.path.join(os.path.dirname(__file__), 'testing-certs', + 'testing.cert') + key_file = os.path.join(os.path.dirname(__file__), 'testing-certs', + 'testing.key') + ssl_context = SSL.Context(SSL.SSLv23_METHOD) + ssl_context.use_privatekey_file(key_file) + ssl_context.use_certificate_chain_file(cert_file) + srv = httpserver.WSGIServerBase(application, host_port, + httpserver.WSGIHandler, + ssl_context=ssl_context + ) + + def shutdown_request(req): + req.shutdown() + srv.close_request(req) + + srv.shutdown_request = shutdown_request + application.base_url = "https://localhost:%s" % srv.server_address[1] + return srv + return make_server, "shutdown", "https" + + +@skip("Skiping tests imported from U1DB.") +class TestHttpSyncTargetHttpsSupport(tests.TestCaseWithServer): + + scenarios = [] + + def setUp(self): + try: + import OpenSSL # noqa + except ImportError: + self.skipTest("Requires pyOpenSSL") + self.cacert_pem = os.path.join(os.path.dirname(__file__), + 'testing-certs', 'cacert.pem') + # The default u1db http_client class for doing HTTPS only does HTTPS + # if the platform is linux. Because of this, soledad replaces that + # class with one that will do HTTPS independent of the platform. In + # order to maintain the compatibility with u1db default tests, we undo + # that replacement here. + http_client._VerifiedHTTPSConnection = \ + soledad.client.api.old__VerifiedHTTPSConnection + super(TestHttpSyncTargetHttpsSupport, self).setUp() + + def getSyncTarget(self, host, path=None, cert_file=None): + if self.server is None: + self.startServer() + return self.sync_target(self, host, path, cert_file=cert_file) + + def test_working(self): + self.startServer() + db = self.request_state._create_database('test') + self.patch(http_client, 'CA_CERTS', self.cacert_pem) + remote_target = self.getSyncTarget('localhost', 'test') + remote_target.record_sync_info('other-id', 2, 'T-id') + self.assertEqual( + (2, 'T-id'), db._get_replica_gen_and_trans_id('other-id')) + + def test_cannot_verify_cert(self): + if not sys.platform.startswith('linux'): + self.skipTest( + "XXX certificate verification happens on linux only for now") + self.startServer() + # don't print expected traceback server-side + self.server.handle_error = lambda req, cli_addr: None + self.request_state._create_database('test') + remote_target = self.getSyncTarget('localhost', 'test') + try: + remote_target.record_sync_info('other-id', 2, 'T-id') + except ssl.SSLError, e: + self.assertIn("certificate verify failed", str(e)) + else: + self.fail("certificate verification should have failed.") + + def test_host_mismatch(self): + if not sys.platform.startswith('linux'): + self.skipTest( + "XXX certificate verification happens on linux only for now") + self.startServer() + self.request_state._create_database('test') + self.patch(http_client, 'CA_CERTS', self.cacert_pem) + remote_target = self.getSyncTarget('127.0.0.1', 'test') + self.assertRaises( + http_client.CertificateError, remote_target.record_sync_info, + 'other-id', 2, 'T-id') + + +load_tests = tests.load_with_scenarios diff --git a/testing/test_soledad/u1db_tests/test_open.py b/testing/test_soledad/u1db_tests/test_open.py new file mode 100644 index 00000000..30d4de00 --- /dev/null +++ b/testing/test_soledad/u1db_tests/test_open.py @@ -0,0 +1,72 @@ +# Copyright 2011 Canonical Ltd. +# Copyright 2016 LEAP Encryption Access Project +# +# This file is part of u1db. +# +# u1db is free software: you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. +# +# u1db is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with u1db. If not, see . + +"""Test u1db.open""" + +import os +from unittest import skip + +from leap.soledad.common.l2db import ( + errors, open as u1db_open, +) +from test_soledad import u1db_tests as tests +from leap.soledad.common.l2db.backends import sqlite_backend +from test_soledad.u1db_tests.test_backends \ + import TestAlternativeDocument + + +@skip("Skiping tests imported from U1DB.") +class TestU1DBOpen(tests.TestCase): + + def setUp(self): + super(TestU1DBOpen, self).setUp() + tmpdir = self.createTempDir() + self.db_path = tmpdir + '/test.db' + + def test_open_no_create(self): + self.assertRaises(errors.DatabaseDoesNotExist, + u1db_open, self.db_path, create=False) + self.assertFalse(os.path.exists(self.db_path)) + + def test_open_create(self): + db = u1db_open(self.db_path, create=True) + self.addCleanup(db.close) + self.assertTrue(os.path.exists(self.db_path)) + self.assertIsInstance(db, sqlite_backend.SQLiteDatabase) + + def test_open_with_factory(self): + db = u1db_open(self.db_path, create=True, + document_factory=TestAlternativeDocument) + self.addCleanup(db.close) + self.assertEqual(TestAlternativeDocument, db._factory) + + def test_open_existing(self): + db = sqlite_backend.SQLitePartialExpandDatabase(self.db_path) + self.addCleanup(db.close) + doc = db.create_doc_from_json(tests.simple_doc) + # Even though create=True, we shouldn't wipe the db + db2 = u1db_open(self.db_path, create=True) + self.addCleanup(db2.close) + doc2 = db2.get_doc(doc.doc_id) + self.assertEqual(doc, doc2) + + def test_open_existing_no_create(self): + db = sqlite_backend.SQLitePartialExpandDatabase(self.db_path) + self.addCleanup(db.close) + db2 = u1db_open(self.db_path, create=False) + self.addCleanup(db2.close) + self.assertIsInstance(db2, sqlite_backend.SQLitePartialExpandDatabase) diff --git a/testing/test_soledad/u1db_tests/testing-certs/Makefile b/testing/test_soledad/u1db_tests/testing-certs/Makefile new file mode 100644 index 00000000..2385e75b --- /dev/null +++ b/testing/test_soledad/u1db_tests/testing-certs/Makefile @@ -0,0 +1,35 @@ +CATOP=./demoCA +ORIG_CONF=/usr/lib/ssl/openssl.cnf +ELEVEN_YEARS=-days 4015 + +init: + cp $(ORIG_CONF) ca.conf + install -d $(CATOP) + install -d $(CATOP)/certs + install -d $(CATOP)/crl + install -d $(CATOP)/newcerts + install -d $(CATOP)/private + touch $(CATOP)/index.txt + echo 01>$(CATOP)/crlnumber + @echo '**** Making CA certificate ...' + openssl req -nodes -new \ + -newkey rsa -keyout $(CATOP)/private/cakey.pem \ + -out $(CATOP)/careq.pem \ + -multivalue-rdn \ + -subj "/C=UK/ST=-/O=u1db LOCAL TESTING ONLY, DO NO TRUST/CN=u1db testing CA" + openssl ca -config ./ca.conf -create_serial \ + -out $(CATOP)/cacert.pem $(ELEVEN_YEARS) -batch \ + -keyfile $(CATOP)/private/cakey.pem -selfsign \ + -extensions v3_ca -infiles $(CATOP)/careq.pem + +pems: + cp ./demoCA/cacert.pem . + openssl req -new -config ca.conf \ + -multivalue-rdn \ + -subj "/O=u1db LOCAL TESTING ONLY, DO NOT TRUST/CN=localhost" \ + -nodes -keyout testing.key -out newreq.pem $(ELEVEN_YEARS) + openssl ca -batch -config ./ca.conf $(ELEVEN_YEARS) \ + -policy policy_anything \ + -out testing.cert -infiles newreq.pem + +.PHONY: init pems diff --git a/testing/test_soledad/u1db_tests/testing-certs/cacert.pem b/testing/test_soledad/u1db_tests/testing-certs/cacert.pem new file mode 100644 index 00000000..c019a730 --- /dev/null +++ b/testing/test_soledad/u1db_tests/testing-certs/cacert.pem @@ -0,0 +1,58 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + e4:de:01:76:c4:78:78:7e + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=UK, ST=-, O=u1db LOCAL TESTING ONLY, DO NO TRUST, CN=u1db testing CA + Validity + Not Before: May 3 11:11:11 2012 GMT + Not After : May 1 11:11:11 2023 GMT + Subject: C=UK, ST=-, O=u1db LOCAL TESTING ONLY, DO NO TRUST, CN=u1db testing CA + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (1024 bit) + Modulus: + 00:bc:91:a5:7f:7d:37:f7:06:c7:db:5b:83:6a:6b: + 63:c3:8b:5c:f7:84:4d:97:6d:d4:be:bf:e7:79:a8: + c1:03:57:ec:90:d4:20:e7:02:95:d9:a6:49:e3:f9: + 9a:ea:37:b9:b2:02:62:ab:40:d3:42:bb:4a:4e:a2: + 47:71:0f:1d:a2:c5:94:a1:cf:35:d3:23:32:42:c0: + 1e:8d:cb:08:58:fb:8a:5c:3e:ea:eb:d5:2c:ed:d6: + aa:09:b4:b5:7d:e3:45:c9:ae:c2:82:b2:ae:c0:81: + bc:24:06:65:a9:e7:e0:61:ac:25:ee:53:d3:d7:be: + 22:f7:00:a2:ad:c6:0e:3a:39 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + DB:3D:93:51:6C:32:15:54:8F:10:50:FC:49:4F:36:15:28:BB:95:6D + X509v3 Authority Key Identifier: + keyid:DB:3D:93:51:6C:32:15:54:8F:10:50:FC:49:4F:36:15:28:BB:95:6D + + X509v3 Basic Constraints: + CA:TRUE + Signature Algorithm: sha1WithRSAEncryption + 72:9b:c1:f7:07:65:83:36:25:4e:01:2f:b7:4a:f2:a4:00:28: + 80:c7:56:2c:32:39:90:13:61:4b:bb:12:c5:44:9d:42:57:85: + 28:19:70:69:e1:43:c8:bd:11:f6:94:df:91:2d:c3:ea:82:8d: + b4:8f:5d:47:a3:00:99:53:29:93:27:6c:c5:da:c1:20:6f:ab: + ec:4a:be:34:f3:8f:02:e5:0c:c0:03:ac:2b:33:41:71:4f:0a: + 72:5a:b4:26:1a:7f:81:bc:c0:95:8a:06:87:a8:11:9f:5c:73: + 38:df:5a:69:40:21:29:ad:46:23:56:75:e1:e9:8b:10:18:4c: + 7b:54 +-----BEGIN CERTIFICATE----- +MIICkjCCAfugAwIBAgIJAOTeAXbEeHh+MA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV +BAYTAlVLMQowCAYDVQQIDAEtMS0wKwYDVQQKDCR1MWRiIExPQ0FMIFRFU1RJTkcg +T05MWSwgRE8gTk8gVFJVU1QxGDAWBgNVBAMMD3UxZGIgdGVzdGluZyBDQTAeFw0x +MjA1MDMxMTExMTFaFw0yMzA1MDExMTExMTFaMGIxCzAJBgNVBAYTAlVLMQowCAYD +VQQIDAEtMS0wKwYDVQQKDCR1MWRiIExPQ0FMIFRFU1RJTkcgT05MWSwgRE8gTk8g +VFJVU1QxGDAWBgNVBAMMD3UxZGIgdGVzdGluZyBDQTCBnzANBgkqhkiG9w0BAQEF +AAOBjQAwgYkCgYEAvJGlf3039wbH21uDamtjw4tc94RNl23Uvr/neajBA1fskNQg +5wKV2aZJ4/ma6je5sgJiq0DTQrtKTqJHcQ8dosWUoc810yMyQsAejcsIWPuKXD7q +69Us7daqCbS1feNFya7CgrKuwIG8JAZlqefgYawl7lPT174i9wCircYOOjkCAwEA +AaNQME4wHQYDVR0OBBYEFNs9k1FsMhVUjxBQ/ElPNhUou5VtMB8GA1UdIwQYMBaA +FNs9k1FsMhVUjxBQ/ElPNhUou5VtMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEF +BQADgYEAcpvB9wdlgzYlTgEvt0rypAAogMdWLDI5kBNhS7sSxUSdQleFKBlwaeFD +yL0R9pTfkS3D6oKNtI9dR6MAmVMpkydsxdrBIG+r7Eq+NPOPAuUMwAOsKzNBcU8K +clq0Jhp/gbzAlYoGh6gRn1xzON9aaUAhKa1GI1Z14emLEBhMe1Q= +-----END CERTIFICATE----- diff --git a/testing/test_soledad/u1db_tests/testing-certs/testing.cert b/testing/test_soledad/u1db_tests/testing-certs/testing.cert new file mode 100644 index 00000000..985684fb --- /dev/null +++ b/testing/test_soledad/u1db_tests/testing-certs/testing.cert @@ -0,0 +1,61 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + e4:de:01:76:c4:78:78:7f + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=UK, ST=-, O=u1db LOCAL TESTING ONLY, DO NO TRUST, CN=u1db testing CA + Validity + Not Before: May 3 11:11:14 2012 GMT + Not After : May 1 11:11:14 2023 GMT + Subject: O=u1db LOCAL TESTING ONLY, DO NOT TRUST, CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (1024 bit) + Modulus: + 00:c6:1d:72:d3:c5:e4:fc:d1:4c:d9:e4:08:3e:90: + 10:ce:3f:1f:87:4a:1d:4f:7f:2a:5a:52:c9:65:4f: + d9:2c:bf:69:75:18:1a:b5:c9:09:32:00:47:f5:60: + aa:c6:dd:3a:87:37:5f:16:be:de:29:b5:ea:fc:41: + 7e:eb:77:bb:df:63:c3:06:1e:ed:e9:a0:67:1a:f1: + ec:e1:9d:f7:9c:8f:1c:fa:c3:66:7b:39:dc:70:ae: + 09:1b:9c:c0:9a:c4:90:77:45:8e:39:95:a9:2f:92: + 43:bd:27:07:5a:99:51:6e:76:a0:af:dd:b1:2c:8f: + ca:8b:8c:47:0d:f6:6e:fc:69 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + Netscape Comment: + OpenSSL Generated Certificate + X509v3 Subject Key Identifier: + 1C:63:85:E1:1D:F3:89:2E:6C:4E:3F:FB:D0:10:64:5A:C1:22:6A:2A + X509v3 Authority Key Identifier: + keyid:DB:3D:93:51:6C:32:15:54:8F:10:50:FC:49:4F:36:15:28:BB:95:6D + + Signature Algorithm: sha1WithRSAEncryption + 1d:6d:3e:bd:93:fd:bd:3e:17:b8:9f:f0:99:7f:db:50:5c:b2: + 01:42:03:b5:d5:94:05:d3:f6:8e:80:82:55:47:1f:58:f2:18: + 6c:ab:ef:43:2c:2f:10:e1:7c:c4:5c:cc:ac:50:50:22:42:aa: + 35:33:f5:b9:f3:a6:66:55:d9:36:f4:f2:e4:d4:d9:b5:2c:52: + 66:d4:21:17:97:22:b8:9b:d7:0e:7c:3d:ce:85:19:ca:c4:d2: + 58:62:31:c6:18:3e:44:fc:f4:30:b6:95:87:ee:21:4a:08:f0: + af:3c:8f:c4:ba:5e:a1:5c:37:1a:7d:7b:fe:66:ae:62:50:17: + 31:ca +-----BEGIN CERTIFICATE----- +MIICnzCCAgigAwIBAgIJAOTeAXbEeHh/MA0GCSqGSIb3DQEBBQUAMGIxCzAJBgNV +BAYTAlVLMQowCAYDVQQIDAEtMS0wKwYDVQQKDCR1MWRiIExPQ0FMIFRFU1RJTkcg +T05MWSwgRE8gTk8gVFJVU1QxGDAWBgNVBAMMD3UxZGIgdGVzdGluZyBDQTAeFw0x +MjA1MDMxMTExMTRaFw0yMzA1MDExMTExMTRaMEQxLjAsBgNVBAoMJXUxZGIgTE9D +QUwgVEVTVElORyBPTkxZLCBETyBOT1QgVFJVU1QxEjAQBgNVBAMMCWxvY2FsaG9z +dDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAxh1y08Xk/NFM2eQIPpAQzj8f +h0odT38qWlLJZU/ZLL9pdRgatckJMgBH9WCqxt06hzdfFr7eKbXq/EF+63e732PD +Bh7t6aBnGvHs4Z33nI8c+sNmeznccK4JG5zAmsSQd0WOOZWpL5JDvScHWplRbnag +r92xLI/Ki4xHDfZu/GkCAwEAAaN7MHkwCQYDVR0TBAIwADAsBglghkgBhvhCAQ0E +HxYdT3BlblNTTCBHZW5lcmF0ZWQgQ2VydGlmaWNhdGUwHQYDVR0OBBYEFBxjheEd +84kubE4/+9AQZFrBImoqMB8GA1UdIwQYMBaAFNs9k1FsMhVUjxBQ/ElPNhUou5Vt +MA0GCSqGSIb3DQEBBQUAA4GBAB1tPr2T/b0+F7if8Jl/21BcsgFCA7XVlAXT9o6A +glVHH1jyGGyr70MsLxDhfMRczKxQUCJCqjUz9bnzpmZV2Tb08uTU2bUsUmbUIReX +Irib1w58Pc6FGcrE0lhiMcYYPkT89DC2lYfuIUoI8K88j8S6XqFcNxp9e/5mrmJQ +FzHK +-----END CERTIFICATE----- diff --git a/testing/test_soledad/u1db_tests/testing-certs/testing.key b/testing/test_soledad/u1db_tests/testing-certs/testing.key new file mode 100644 index 00000000..d83d4920 --- /dev/null +++ b/testing/test_soledad/u1db_tests/testing-certs/testing.key @@ -0,0 +1,16 @@ +-----BEGIN PRIVATE KEY----- +MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAMYdctPF5PzRTNnk +CD6QEM4/H4dKHU9/KlpSyWVP2Sy/aXUYGrXJCTIAR/VgqsbdOoc3Xxa+3im16vxB +fut3u99jwwYe7emgZxrx7OGd95yPHPrDZns53HCuCRucwJrEkHdFjjmVqS+SQ70n +B1qZUW52oK/dsSyPyouMRw32bvxpAgMBAAECgYBs3lXxhjg1rhabTjIxnx19GTcM +M3Az9V+izweZQu3HJ1CeZiaXauhAr+LbNsniCkRVddotN6oCJdQB10QVxXBZc9Jz +HPJ4zxtZfRZlNMTMmG7eLWrfxpgWnb/BUjDb40yy1nhr9yhDUnI/8RoHDRHnAEHZ +/CnHGUrqcVcrY5zJAQJBAPLhBJg9W88JVmcOKdWxRgs7dLHnZb999Kv1V5mczmAi +jvGvbUmucqOqke6pTUHNYyNHqU6pySzGUi2cH+BAkFECQQDQ0VoAOysg6FVoT15v +tGh57t5sTiCZZ7PS8jwvtThsgA+vcf6c16XWzXgjGXSap4r2QDOY2rI5lsWLaQ8T ++fyZAkAfyFJRmbXp4c7srW3MCOahkaYzoZQu+syJtBFCiMJ40gzik5I5khpuUGPI +V19EvRu8AiSlppIsycb3MPb64XgBAkEAy7DrUf5le5wmc7G4NM6OeyJ+5LbxJbL6 +vnJ8My1a9LuWkVVpQCU7J+UVo2dZTuLPspW9vwTVhUeFOxAoHRxlQQJAFem93f7m +el2BkB2EFqU3onPejkZ5UrDmfmeOQR1axMQNSXqSxcJxqa16Ru1BWV2gcWRbwajQ +oc+kuJThu/r/Ug== +-----END PRIVATE KEY----- diff --git a/testing/test_soledad/util.py b/testing/test_soledad/util.py new file mode 100644 index 00000000..f81001b9 --- /dev/null +++ b/testing/test_soledad/util.py @@ -0,0 +1,430 @@ +# -*- CODING: UTF-8 -*- +# util.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +Utilities used by multiple test suites. +""" + + +import os +import tempfile +import shutil +import random +import string +import couchdb + +from uuid import uuid4 +from mock import Mock +from urlparse import urljoin +from StringIO import StringIO +from pysqlcipher import dbapi2 + +from twisted.trial import unittest + +from leap.common.testing.basetest import BaseLeapTest + +from leap.soledad.common import l2db +from leap.soledad.common.l2db import sync +from leap.soledad.common.l2db.remote import http_database + +from leap.soledad.common import soledad_assert +from leap.soledad.common.document import SoledadDocument +from leap.soledad.common.couch import CouchDatabase +from leap.soledad.common.couch.state import CouchServerState + +from leap.soledad.common.crypto import ENC_SCHEME_KEY + +from leap.soledad.client import Soledad +from leap.soledad.client import http_target +from leap.soledad.client import auth +from leap.soledad.client.crypto import decrypt_doc_dict +from leap.soledad.client.sqlcipher import SQLCipherDatabase +from leap.soledad.client.sqlcipher import SQLCipherOptions + +from leap.soledad.server import SoledadApp +from leap.soledad.server.auth import SoledadTokenAuthMiddleware + + +PASSWORD = '123456' +ADDRESS = 'leap@leap.se' + + +def make_local_db_and_target(test): + db = test.create_database('test') + st = db.get_sync_target() + return db, st + + +def make_document_for_test(test, doc_id, rev, content, has_conflicts=False): + return SoledadDocument(doc_id, rev, content, has_conflicts=has_conflicts) + + +def make_sqlcipher_database_for_test(test, replica_uid): + db = SQLCipherDatabase( + SQLCipherOptions(':memory:', PASSWORD)) + db._set_replica_uid(replica_uid) + return db + + +def copy_sqlcipher_database_for_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR + # HOUSE. + new_db = make_sqlcipher_database_for_test(test, None) + tmpfile = StringIO() + for line in db._db_handle.iterdump(): + if 'sqlite_sequence' not in line: # work around bug in iterdump + tmpfile.write('%s\n' % line) + tmpfile.seek(0) + new_db._db_handle = dbapi2.connect(':memory:') + new_db._db_handle.cursor().executescript(tmpfile.read()) + new_db._db_handle.commit() + new_db._set_replica_uid(db._replica_uid) + new_db._factory = db._factory + return new_db + + +SQLCIPHER_SCENARIOS = [ + ('sqlcipher', {'make_database_for_test': make_sqlcipher_database_for_test, + 'copy_database_for_test': copy_sqlcipher_database_for_test, + 'make_document_for_test': make_document_for_test, }), +] + + +def make_soledad_app(state): + return SoledadApp(state) + + +def make_token_soledad_app(state): + app = SoledadApp(state) + + def _verify_authentication_data(uuid, auth_data): + if uuid.startswith('user-') and auth_data == 'auth-token': + return True + return False + + # we test for action authorization in leap.soledad.common.tests.test_server + def _verify_authorization(uuid, environ): + return True + + application = SoledadTokenAuthMiddleware(app) + application._verify_authentication_data = _verify_authentication_data + application._verify_authorization = _verify_authorization + return application + + +def make_soledad_document_for_test(test, doc_id, rev, content, + has_conflicts=False): + return SoledadDocument( + doc_id, rev, content, has_conflicts=has_conflicts) + + +def make_token_http_database_for_test(test, replica_uid): + test.startServer() + test.request_state._create_database(replica_uid) + + class _HTTPDatabaseWithToken( + http_database.HTTPDatabase, auth.TokenBasedAuth): + + def set_token_credentials(self, uuid, token): + auth.TokenBasedAuth.set_token_credentials(self, uuid, token) + + def _sign_request(self, method, url_query, params): + return auth.TokenBasedAuth._sign_request( + self, method, url_query, params) + + http_db = _HTTPDatabaseWithToken(test.getURL('test')) + http_db.set_token_credentials('user-uuid', 'auth-token') + return http_db + + +def copy_token_http_database_for_test(test, db): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES IS + # THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST THAT WE + # CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS RATHER THAN + # CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND NINJA TO YOUR + # HOUSE. + http_db = test.request_state._copy_database(db) + http_db.set_token_credentials(http_db, 'user-uuid', 'auth-token') + return http_db + + +def sync_via_synchronizer(test, db_source, db_target, trace_hook=None, + trace_hook_shallow=None): + target = db_target.get_sync_target() + trace_hook = trace_hook or trace_hook_shallow + if trace_hook: + target._set_trace_hook(trace_hook) + return sync.Synchronizer(db_source, target).sync() + + +class MockedSharedDBTest(object): + + def get_default_shared_mock(self, put_doc_side_effect=None, + get_doc_return_value=None): + """ + Get a default class for mocking the shared DB + """ + class defaultMockSharedDB(object): + get_doc = Mock(return_value=get_doc_return_value) + put_doc = Mock(side_effect=put_doc_side_effect) + open = Mock(return_value=None) + close = Mock(return_value=None) + syncable = True + + def __call__(self): + return self + return defaultMockSharedDB + + +def soledad_sync_target( + test, path, source_replica_uid=uuid4().hex, + sync_db=None, sync_enc_pool=None): + creds = {'token': { + 'uuid': 'user-uuid', + 'token': 'auth-token', + }} + return http_target.SoledadHTTPSyncTarget( + test.getURL(path), + source_replica_uid, + creds, + test._soledad._crypto, + None, # cert_file + sync_db=sync_db, + sync_enc_pool=sync_enc_pool) + + +# redefine the base leap test class so it inherits from twisted trial's +# TestCase. This is needed so trial knows that it has to manage a reactor and +# wait for deferreds returned by tests to be fired. +BaseLeapTest = type( + 'BaseLeapTest', (unittest.TestCase,), dict(BaseLeapTest.__dict__)) + + +class BaseSoledadTest(BaseLeapTest, MockedSharedDBTest): + + """ + Instantiates Soledad for usage in tests. + """ + defer_sync_encryption = False + + def setUp(self): + # The following snippet comes from BaseLeapTest.setUpClass, but we + # repeat it here because twisted.trial does not work with + # setUpClass/tearDownClass. + + self.old_path = os.environ['PATH'] + self.old_home = os.environ['HOME'] + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + self.home = self.tempdir + bin_tdir = os.path.join( + self.tempdir, + 'bin') + os.environ["PATH"] = bin_tdir + os.environ["HOME"] = self.tempdir + + # config info + self.db1_file = os.path.join(self.tempdir, "db1.u1db") + self.db2_file = os.path.join(self.tempdir, "db2.u1db") + self.email = ADDRESS + # open test dbs + self._db1 = l2db.open(self.db1_file, create=True, + document_factory=SoledadDocument) + self._db2 = l2db.open(self.db2_file, create=True, + document_factory=SoledadDocument) + # get a random prefix for each test, so we do not mess with + # concurrency during initialization and shutting down of + # each local db. + self.rand_prefix = ''.join( + map(lambda x: random.choice(string.ascii_letters), range(6))) + + # initialize soledad by hand so we can control keys + # XXX check if this soledad is actually used + self._soledad = self._soledad_instance( + prefix=self.rand_prefix, user=self.email) + + def tearDown(self): + self._db1.close() + self._db2.close() + self._soledad.close() + + # restore paths + os.environ["PATH"] = self.old_path + os.environ["HOME"] = self.old_home + + def _delete_temporary_dirs(): + # XXX should not access "private" attrs + for f in [self._soledad.local_db_path, + self._soledad.secrets.secrets_path]: + if os.path.isfile(f): + os.unlink(f) + # The following snippet comes from BaseLeapTest.setUpClass, but we + # repeat it here because twisted.trial does not work with + # setUpClass/tearDownClass. + soledad_assert( + self.tempdir.startswith('/tmp/leap_tests-'), + "beware! tried to remove a dir which does not " + "live in temporal folder!") + shutil.rmtree(self.tempdir) + + from twisted.internet import reactor + reactor.addSystemEventTrigger( + "after", "shutdown", _delete_temporary_dirs) + + def _soledad_instance(self, user=ADDRESS, passphrase=u'123', + prefix='', + secrets_path='secrets.json', + local_db_path='soledad.u1db', + server_url='https://127.0.0.1/', + cert_file=None, + shared_db_class=None, + auth_token='auth-token'): + + def _put_doc_side_effect(doc): + self._doc_put = doc + + if shared_db_class is not None: + MockSharedDB = shared_db_class + else: + MockSharedDB = self.get_default_shared_mock( + _put_doc_side_effect) + + soledad = Soledad( + user, + passphrase, + secrets_path=os.path.join( + self.tempdir, prefix, secrets_path), + local_db_path=os.path.join( + self.tempdir, prefix, local_db_path), + server_url=server_url, # Soledad will fail if not given an url. + cert_file=cert_file, + defer_encryption=self.defer_sync_encryption, + shared_db=MockSharedDB(), + auth_token=auth_token) + self.addCleanup(soledad.close) + return soledad + + def assertGetEncryptedDoc( + self, db, doc_id, doc_rev, content, has_conflicts): + """ + Assert that the document in the database looks correct. + """ + exp_doc = self.make_document(doc_id, doc_rev, content, + has_conflicts=has_conflicts) + doc = db.get_doc(doc_id) + + if ENC_SCHEME_KEY in doc.content: + # XXX check for SYM_KEY too + key = self._soledad._crypto.doc_passphrase(doc.doc_id) + secret = self._soledad._crypto.secret + decrypted = decrypt_doc_dict( + doc.content, doc.doc_id, doc.rev, + key, secret) + doc.set_json(decrypted) + self.assertEqual(exp_doc.doc_id, doc.doc_id) + self.assertEqual(exp_doc.rev, doc.rev) + self.assertEqual(exp_doc.has_conflicts, doc.has_conflicts) + self.assertEqual(exp_doc.content, doc.content) + + +class CouchDBTestCase(unittest.TestCase, MockedSharedDBTest): + + """ + TestCase base class for tests against a real CouchDB server. + """ + + def setUp(self): + """ + Make sure we have a CouchDB instance for a test. + """ + self.couch_port = 5984 + self.couch_url = 'http://localhost:%d' % self.couch_port + self.couch_server = couchdb.Server(self.couch_url) + + def delete_db(self, name): + try: + self.couch_server.delete(name) + except: + # ignore if already missing + pass + + +class CouchServerStateForTests(CouchServerState): + + """ + This is a slightly modified CouchDB server state that allows for creating + a database. + + Ordinarily, the CouchDB server state does not allow some operations, + because for security purposes the Soledad Server should not even have + enough permissions to perform them. For tests, we allow database creation, + otherwise we'd have to create those databases in setUp/tearDown methods, + which is less pleasant than allowing the db to be automatically created. + """ + + def __init__(self, *args, **kwargs): + self.dbs = [] + super(CouchServerStateForTests, self).__init__(*args, **kwargs) + + def _create_database(self, replica_uid=None, dbname=None): + """ + Create db and append to a list, allowing test to close it later + """ + dbname = dbname or ('test-%s' % uuid4().hex) + db = CouchDatabase.open_database( + urljoin(self.couch_url, dbname), + True, + replica_uid=replica_uid or 'test', + ensure_ddocs=True) + self.dbs.append(db) + return db + + def ensure_database(self, dbname): + db = self._create_database(dbname=dbname) + return db, db.replica_uid + + +class SoledadWithCouchServerMixin( + BaseSoledadTest, + CouchDBTestCase): + + def setUp(self): + CouchDBTestCase.setUp(self) + BaseSoledadTest.setUp(self) + main_test_class = getattr(self, 'main_test_class', None) + if main_test_class is not None: + main_test_class.setUp(self) + + def tearDown(self): + main_test_class = getattr(self, 'main_test_class', None) + if main_test_class is not None: + main_test_class.tearDown(self) + # delete the test database + BaseSoledadTest.tearDown(self) + CouchDBTestCase.tearDown(self) + + def make_app(self): + self.request_state = CouchServerStateForTests(self.couch_url) + self.addCleanup(self.delete_dbs) + return self.make_app_with_state(self.request_state) + + def delete_dbs(self): + for db in self.request_state.dbs: + self.delete_db(db._dbname) diff --git a/testing/tests/client/__init__.py b/testing/tests/client/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testing/tests/client/hacker_crackdown.txt b/testing/tests/client/hacker_crackdown.txt new file mode 100644 index 00000000..a01eb509 --- /dev/null +++ b/testing/tests/client/hacker_crackdown.txt @@ -0,0 +1,13005 @@ +The Project Gutenberg EBook of Hacker Crackdown, by Bruce Sterling + +This eBook is for the use of anyone anywhere at no cost and with +almost no restrictions whatsoever. You may copy it, give it away or +re-use it under the terms of the Project Gutenberg License included +with this eBook or online at www.gutenberg.org + +** This is a COPYRIGHTED Project Gutenberg eBook, Details Below ** +** Please follow the copyright guidelines in this file. ** + +Title: Hacker Crackdown + Law and Disorder on the Electronic Frontier + +Author: Bruce Sterling + +Posting Date: February 9, 2012 [EBook #101] +Release Date: January, 1994 + +Language: English + + +*** START OF THIS PROJECT GUTENBERG EBOOK HACKER CRACKDOWN *** + + + + + + + + + + + + + +THE HACKER CRACKDOWN + +Law and Disorder on the Electronic Frontier + +by Bruce Sterling + + + + +CONTENTS + + +Preface to the Electronic Release of The Hacker Crackdown + +Chronology of the Hacker Crackdown + + +Introduction + + +Part 1: CRASHING THE SYSTEM +A Brief History of Telephony +Bell's Golden Vaporware +Universal Service +Wild Boys and Wire Women +The Electronic Communities +The Ungentle Giant +The Breakup +In Defense of the System +The Crash Post-Mortem +Landslides in Cyberspace + + +Part 2: THE DIGITAL UNDERGROUND +Steal This Phone +Phreaking and Hacking +The View From Under the Floorboards +Boards: Core of the Underground +Phile Phun +The Rake's Progress +Strongholds of the Elite +Sting Boards +Hot Potatoes +War on the Legion +Terminus +Phile 9-1-1 +War Games +Real Cyberpunk + + +Part 3: LAW AND ORDER +Crooked Boards +The World's Biggest Hacker Bust +Teach Them a Lesson +The U.S. Secret Service +The Secret Service Battles the Boodlers +A Walk Downtown +FCIC: The Cutting-Edge Mess +Cyberspace Rangers +FLETC: Training the Hacker-Trackers + + +Part 4: THE CIVIL LIBERTARIANS +NuPrometheus + FBI = Grateful Dead +Whole Earth + Computer Revolution = WELL +Phiber Runs Underground and Acid Spikes the Well +The Trial of Knight Lightning +Shadowhawk Plummets to Earth +Kyrie in the Confessional +$79,499 +A Scholar Investigates +Computers, Freedom, and Privacy + + +Electronic Afterword to The Hacker Crackdown, Halloween 1993 + + + + +THE HACKER CRACKDOWN + +Law and Disorder on the Electronic Frontier + +by Bruce Sterling + + + + + +Preface to the Electronic Release of The Hacker Crackdown + + +January 1, 1994--Austin, Texas + + +Hi, I'm Bruce Sterling, the author of this electronic book. + +Out in the traditional world of print, The Hacker Crackdown +is ISBN 0-553-08058-X, and is formally catalogued by +the Library of Congress as "1. Computer crimes--United States. +2. Telephone--United States--Corrupt practices. +3. Programming (Electronic computers)--United States--Corrupt practices." + +`Corrupt practices,' I always get a kick out of that description. +Librarians are very ingenious people. + +The paperback is ISBN 0-553-56370-X. If you go +and buy a print version of The Hacker Crackdown, +an action I encourage heartily, you may notice that +in the front of the book, beneath the copyright notice-- +"Copyright (C) 1992 by Bruce Sterling"-- +it has this little block of printed legal +boilerplate from the publisher. It says, and I quote: + + "No part of this book may be reproduced or transmitted in any form +or by any means, electronic or mechanical, including photocopying, +recording, or by any information storage and retrieval system, +without permission in writing from the publisher. +For information address: Bantam Books." + +This is a pretty good disclaimer, as such disclaimers go. +I collect intellectual-property disclaimers, and I've seen dozens of them, +and this one is at least pretty straightforward. In this narrow +and particular case, however, it isn't quite accurate. +Bantam Books puts that disclaimer on every book they publish, +but Bantam Books does not, in fact, own the electronic rights to this book. +I do, because of certain extensive contract maneuverings my agent and I +went through before this book was written. I want to give those electronic +publishing rights away through certain not-for-profit channels, +and I've convinced Bantam that this is a good idea. + +Since Bantam has seen fit to peacably agree to this scheme of mine, +Bantam Books is not going to fuss about this. Provided you don't try +to sell the book, they are not going to bother you for what you do with +the electronic copy of this book. If you want to check this out personally, +you can ask them; they're at 1540 Broadway NY NY 10036. However, if you were +so foolish as to print this book and start retailing it for money in violation +of my copyright and the commercial interests of Bantam Books, then Bantam, +a part of the gigantic Bertelsmann multinational publishing combine, +would roust some of their heavy-duty attorneys out of hibernation +and crush you like a bug. This is only to be expected. +I didn't write this book so that you could make money out of it. +If anybody is gonna make money out of this book, +it's gonna be me and my publisher. + +My publisher deserves to make money out of this book. +Not only did the folks at Bantam Books commission me +to write the book, and pay me a hefty sum to do so, but +they bravely printed, in text, an electronic document the +reproduction of which was once alleged to be a federal felony. +Bantam Books and their numerous attorneys were very brave +and forthright about this book. Furthermore, my former editor +at Bantam Books, Betsy Mitchell, genuinely cared about this project, +and worked hard on it, and had a lot of wise things to say +about the manuscript. Betsy deserves genuine credit for this book, +credit that editors too rarely get. + +The critics were very kind to The Hacker Crackdown, +and commercially the book has done well. On the other hand, +I didn't write this book in order to squeeze every last nickel +and dime out of the mitts of impoverished sixteen-year-old +cyberpunk high-school-students. Teenagers don't have any money-- +(no, not even enough for the six-dollar Hacker Crackdown paperback, +with its attractive bright-red cover and useful index). +That's a major reason why teenagers sometimes succumb to the temptation +to do things they shouldn't, such as swiping my books out of libraries. +Kids: this one is all yours, all right? Go give the print version back. +*8-) + +Well-meaning, public-spirited civil libertarians don't have much money, +either. And it seems almost criminal to snatch cash out of the hands of +America's direly underpaid electronic law enforcement community. + +If you're a computer cop, a hacker, or an electronic civil +liberties activist, you are the target audience for this book. +I wrote this book because I wanted to help you, and help other people +understand you and your unique, uhm, problems. I wrote this book +to aid your activities, and to contribute to the public discussion +of important political issues. In giving the text away in this +fashion, I am directly contributing to the book's ultimate aim: +to help civilize cyberspace. + +Information WANTS to be free. And the information inside +this book longs for freedom with a peculiar intensity. +I genuinely believe that the natural habitat of this book +is inside an electronic network. That may not be the easiest +direct method to generate revenue for the book's author, +but that doesn't matter; this is where this book belongs +by its nature. I've written other books--plenty of other books-- +and I'll write more and I am writing more, but this one is special. +I am making The Hacker Crackdown available electronically +as widely as I can conveniently manage, and if you like the book, +and think it is useful, then I urge you to do the same with it. + +You can copy this electronic book. Copy the heck out of it, +be my guest, and give those copies to anybody who wants them. +The nascent world of cyberspace is full of sysadmins, teachers, +trainers, cybrarians, netgurus, and various species of cybernetic activist. +If you're one of those people, I know about you, and I know the hassle +you go through to try to help people learn about the electronic frontier. +I hope that possessing this book in electronic form will lessen your troubles. +Granted, this treatment of our electronic social spectrum is not the ultimate +in academic rigor. And politically, it has something to offend +and trouble almost everyone. But hey, I'm told it's readable, +and at least the price is right. + +You can upload the book onto bulletin board systems, or Internet nodes, +or electronic discussion groups. Go right ahead and do that, I am giving +you express permission right now. Enjoy yourself. + +You can put the book on disks and give the disks away, +as long as you don't take any money for it. + +But this book is not public domain. You can't copyright it in +your own name. I own the copyright. Attempts to pirate this book +and make money from selling it may involve you in a serious litigative snarl. +Believe me, for the pittance you might wring out of such an action, +it's really not worth it. This book don't "belong" to you. +In an odd but very genuine way, I feel it doesn't "belong" to me, either. +It's a book about the people of cyberspace, and distributing it in this way +is the best way I know to actually make this information available, +freely and easily, to all the people of cyberspace--including people +far outside the borders of the United States, who otherwise may never +have a chance to see any edition of the book, and who may perhaps learn +something useful from this strange story of distant, obscure, but portentous +events in so-called "American cyberspace." + +This electronic book is now literary freeware. It now belongs to the +emergent realm of alternative information economics. You have no right +to make this electronic book part of the conventional flow of commerce. +Let it be part of the flow of knowledge: there's a difference. +I've divided the book into four sections, so that it is less ungainly +for upload and download; if there's a section of particular relevance +to you and your colleagues, feel free to reproduce that one and skip the rest. + +[Project Gutenberg has reassembled the file, with Sterling's permission.] + +Just make more when you need them, and give them to whoever might want them. + +Now have fun. + +Bruce Sterling--bruces@well.sf.ca.us + + +THE HACKER CRACKDOWN + +Law and Disorder on the Electronic Frontier + +by Bruce Sterling + + + + + + + +CHRONOLOGY OF THE HACKER CRACKDOWN + + +1865 U.S. Secret Service (USSS) founded. + +1876 Alexander Graham Bell invents telephone. + +1878 First teenage males flung off phone system by enraged authorities. + +1939 "Futurian" science-fiction group raided by Secret Service. + +1971 Yippie phone phreaks start YIPL/TAP magazine. + +1972 RAMPARTS magazine seized in blue-box rip-off scandal. + +1978 Ward Christenson and Randy Suess create first personal + computer bulletin board system. + +1982 William Gibson coins term "cyberspace." + +1982 "414 Gang" raided. + +1983-1983 AT&T dismantled in divestiture. + +1984 Congress passes Comprehensive Crime Control Act giving USSS + jurisdiction over credit card fraud and computer fraud. + +1984 "Legion of Doom" formed. + +1984. 2600: THE HACKER QUARTERLY founded. + +1984. WHOLE EARTH SOFTWARE CATALOG published. + +1985. First police "sting" bulletin board systems established. + +1985. Whole Earth 'Lectronic Link computer conference (WELL) goes on-line. + +1986 Computer Fraud and Abuse Act passed. + +1986 Electronic Communications Privacy Act passed. + +1987 Chicago prosecutors form Computer Fraud and Abuse Task Force. + + +1988 + +July. Secret Service covertly videotapes "SummerCon" hacker convention. + +September. "Prophet" cracks BellSouth AIMSX computer network + and downloads E911 Document to his own computer and to Jolnet. + +September. AT&T Corporate Information Security informed of Prophet's action. + +October. Bellcore Security informed of Prophet's action. + + +1989 + +January. Prophet uploads E911 Document to Knight Lightning. + +February 25. Knight Lightning publishes E911 Document in PHRACK + electronic newsletter. + +May. Chicago Task Force raids and arrests "Kyrie." + +June. "NuPrometheus League" distributes Apple Computer proprietary software. + +June 13. Florida probation office crossed with phone-sex line + in switching-station stunt. + +July. "Fry Guy" raided by USSS and Chicago Computer Fraud + and Abuse Task Force. + +July. Secret Service raids "Prophet," "Leftist," and "Urvile" in Georgia. + + +1990 + +January 15. Martin Luther King Day Crash strikes AT&T long-distance + network nationwide. + +January 18-19. Chicago Task Force raids Knight Lightning in St. Louis. + +January 24. USSS and New York State Police raid "Phiber Optik," + "Acid Phreak," and "Scorpion" in New York City. + +February 1. USSS raids "Terminus" in Maryland. + +February 3. Chicago Task Force raids Richard Andrews' home. + +February 6. Chicago Task Force raids Richard Andrews' business. + +February 6. USSS arrests Terminus, Prophet, Leftist, and Urvile. + +February 9. Chicago Task Force arrests Knight Lightning. + +February 20. AT&T Security shuts down public-access + "attctc" computer in Dallas. + +February 21. Chicago Task Force raids Robert Izenberg in Austin. + +March 1. Chicago Task Force raids Steve Jackson Games, Inc., + "Mentor," and "Erik Bloodaxe" in Austin. + +May 7,8,9. + +USSS and Arizona Organized Crime and Racketeering Bureau conduct +"Operation Sundevil" raids in Cincinnatti, Detroit, Los Angeles, +Miami, Newark, Phoenix, Pittsburgh, Richmond, Tucson, San Diego, +San Jose, and San Francisco. + +May. FBI interviews John Perry Barlow re NuPrometheus case. + +June. Mitch Kapor and Barlow found Electronic Frontier Foundation; + Barlow publishes CRIME AND PUZZLEMENT manifesto. + +July 24-27. Trial of Knight Lightning. + +1991 + +February. CPSR Roundtable in Washington, D.C. + +March 25-28. Computers, Freedom and Privacy conference in San Francisco. + +May 1. Electronic Frontier Foundation, Steve Jackson, + and others file suit against members of Chicago Task Force. + +July 1-2. Switching station phone software crash affects + Washington, Los Angeles, Pittsburgh, San Francisco. + +September 17. AT&T phone crash affects New York City and three airports. + + + + +Introduction + +This is a book about cops, and wild teenage whiz-kids, and lawyers, +and hairy-eyed anarchists, and industrial technicians, and hippies, +and high-tech millionaires, and game hobbyists, and computer security +experts, and Secret Service agents, and grifters, and thieves. + +This book is about the electronic frontier of the 1990s. +It concerns activities that take place inside computers +and over telephone lines. + +A science fiction writer coined the useful term "cyberspace" in 1982, +but the territory in question, the electronic frontier, is about +a hundred and thirty years old. Cyberspace is the "place" where +a telephone conversation appears to occur. Not inside your actual phone, +the plastic device on your desk. Not inside the other person's phone, +in some other city. THE PLACE BETWEEN the phones. The indefinite +place OUT THERE, where the two of you, two human beings, +actually meet and communicate. + +Although it is not exactly "real," "cyberspace" is a genuine place. +Things happen there that have very genuine consequences. This "place" +is not "real," but it is serious, it is earnest. Tens of thousands +of people have dedicated their lives to it, to the public service +of public communication by wire and electronics. + +People have worked on this "frontier" for generations now. +Some people became rich and famous from their efforts there. +Some just played in it, as hobbyists. Others soberly pondered it, +and wrote about it, and regulated it, and negotiated over it in +international forums, and sued one another about it, in gigantic, +epic court battles that lasted for years. And almost since +the beginning, some people have committed crimes in this place. + +But in the past twenty years, this electrical "space," +which was once thin and dark and one-dimensional--little more +than a narrow speaking-tube, stretching from phone to phone-- +has flung itself open like a gigantic jack-in-the-box. +Light has flooded upon it, the eerie light of the glowing computer screen. +This dark electric netherworld has become a vast flowering electronic landscape. +Since the 1960s, the world of the telephone has cross-bred itself +with computers and television, and though there is still no substance +to cyberspace, nothing you can handle, it has a strange kind +of physicality now. It makes good sense today to talk of cyberspace +as a place all its own. + +Because people live in it now. Not just a few people, +not just a few technicians and eccentrics, but thousands +of people, quite normal people. And not just for a little while, +either, but for hours straight, over weeks, and months, +and years. Cyberspace today is a "Net," a "Matrix," +international in scope and growing swiftly and steadily. +It's growing in size, and wealth, and political importance. + +People are making entire careers in modern cyberspace. +Scientists and technicians, of course; they've been there +for twenty years now. But increasingly, cyberspace +is filling with journalists and doctors and lawyers +and artists and clerks. Civil servants make their +careers there now, "on-line" in vast government data-banks; +and so do spies, industrial, political, and just plain snoops; +and so do police, at least a few of them. And there are children +living there now. + +People have met there and been married there. +There are entire living communities in cyberspace today; +chattering, gossiping, planning, conferring and scheming, +leaving one another voice-mail and electronic mail, +giving one another big weightless chunks of valuable data, +both legitimate and illegitimate. They busily pass one another +computer software and the occasional festering computer virus. + +We do not really understand how to live in cyberspace yet. +We are feeling our way into it, blundering about. +That is not surprising. Our lives in the physical world, +the "real" world, are also far from perfect, despite a lot more practice. +Human lives, real lives, are imperfect by their nature, and there are +human beings in cyberspace. The way we live in cyberspace is +a funhouse mirror of the way we live in the real world. +We take both our advantages and our troubles with us. + +This book is about trouble in cyberspace. +Specifically, this book is about certain strange events in +the year 1990, an unprecedented and startling year for the +the growing world of computerized communications. + +In 1990 there came a nationwide crackdown on illicit +computer hackers, with arrests, criminal charges, +one dramatic show-trial, several guilty pleas, and +huge confiscations of data and equipment all over the USA. + +The Hacker Crackdown of 1990 was larger, better organized, +more deliberate, and more resolute than any previous effort +in the brave new world of computer crime. The U.S. Secret Service, +private telephone security, and state and local law enforcement groups +across the country all joined forces in a determined attempt to break +the back of America's electronic underground. It was a fascinating +effort, with very mixed results. + +The Hacker Crackdown had another unprecedented effect; +it spurred the creation, within "the computer community," +of the Electronic Frontier Foundation, a new and very odd +interest group, fiercely dedicated to the establishment +and preservation of electronic civil liberties. The crackdown, +remarkable in itself, has created a melee of debate over electronic crime, +punishment, freedom of the press, and issues of search and seizure. +Politics has entered cyberspace. Where people go, politics follow. + +This is the story of the people of cyberspace. + + + +PART ONE: Crashing the System + +On January 15, 1990, AT&T's long-distance telephone switching system crashed. + +This was a strange, dire, huge event. Sixty thousand people lost +their telephone service completely. During the nine long hours +of frantic effort that it took to restore service, some seventy million +telephone calls went uncompleted. + +Losses of service, known as "outages" in the telco trade, +are a known and accepted hazard of the telephone business. +Hurricanes hit, and phone cables get snapped by the thousands. +Earthquakes wrench through buried fiber-optic lines. +Switching stations catch fire and burn to the ground. +These things do happen. There are contingency plans for them, +and decades of experience in dealing with them. +But the Crash of January 15 was unprecedented. +It was unbelievably huge, and it occurred for +no apparent physical reason. + +The crash started on a Monday afternoon in a single +switching-station in Manhattan. But, unlike any merely +physical damage, it spread and spread. Station after +station across America collapsed in a chain reaction, +until fully half of AT&T's network had gone haywire +and the remaining half was hard-put to handle the overflow. + +Within nine hours, AT&T software engineers more or less +understood what had caused the crash. Replicating the +problem exactly, poring over software line by line, +took them a couple of weeks. But because it was hard +to understand technically, the full truth of the matter +and its implications were not widely and thoroughly aired +and explained. The root cause of the crash remained obscure, +surrounded by rumor and fear. + +The crash was a grave corporate embarrassment. +The "culprit" was a bug in AT&T's own software--not the +sort of admission the telecommunications giant wanted +to make, especially in the face of increasing competition. +Still, the truth WAS told, in the baffling technical terms +necessary to explain it. + +Somehow the explanation failed to persuade +American law enforcement officials and even telephone +corporate security personnel. These people were not +technical experts or software wizards, and they had their +own suspicions about the cause of this disaster. + +The police and telco security had important sources +of information denied to mere software engineers. +They had informants in the computer underground and +years of experience in dealing with high-tech rascality +that seemed to grow ever more sophisticated. +For years they had been expecting a direct and +savage attack against the American national telephone system. +And with the Crash of January 15--the first month of a +new, high-tech decade--their predictions, fears, +and suspicions seemed at last to have entered the real world. +A world where the telephone system had not merely crashed, +but, quite likely, BEEN crashed--by "hackers." + +The crash created a large dark cloud of suspicion +that would color certain people's assumptions and actions +for months. The fact that it took place in the realm of +software was suspicious on its face. The fact that it +occurred on Martin Luther King Day, still the most +politically touchy of American holidays, made it more +suspicious yet. + +The Crash of January 15 gave the Hacker Crackdown +its sense of edge and its sweaty urgency. It made people, +powerful people in positions of public authority, +willing to believe the worst. And, most fatally, +it helped to give investigators a willingness +to take extreme measures and the determination +to preserve almost total secrecy. + +An obscure software fault in an aging switching system +in New York was to lead to a chain reaction of legal +and constitutional trouble all across the country. + +# + +Like the crash in the telephone system, this chain reaction +was ready and waiting to happen. During the 1980s, +the American legal system was extensively patched +to deal with the novel issues of computer crime. +There was, for instance, the Electronic Communications +Privacy Act of 1986 (eloquently described as "a stinking mess" +by a prominent law enforcement official). And there was the +draconian Computer Fraud and Abuse Act of 1986, passed unanimously +by the United States Senate, which later would reveal +a large number of flaws. Extensive, well-meant efforts +had been made to keep the legal system up to date. +But in the day-to-day grind of the real world, +even the most elegant software tends to crumble +and suddenly reveal its hidden bugs. + +Like the advancing telephone system, the American legal system +was certainly not ruined by its temporary crash; but for those +caught under the weight of the collapsing system, life became +a series of blackouts and anomalies. + +In order to understand why these weird events occurred, +both in the world of technology and in the world of law, +it's not enough to understand the merely technical problems. +We will get to those; but first and foremost, we must try +to understand the telephone, and the business of telephones, +and the community of human beings that telephones have created. + +# + +Technologies have life cycles, like cities do, +like institutions do, like laws and governments do. + +The first stage of any technology is the Question +Mark, often known as the "Golden Vaporware" stage. +At this early point, the technology is only a phantom, +a mere gleam in the inventor's eye. One such inventor +was a speech teacher and electrical tinkerer named +Alexander Graham Bell. + +Bell's early inventions, while ingenious, failed to move the world. +In 1863, the teenage Bell and his brother Melville made an artificial +talking mechanism out of wood, rubber, gutta-percha, and tin. +This weird device had a rubber-covered "tongue" made of movable +wooden segments, with vibrating rubber "vocal cords," and +rubber "lips" and "cheeks." While Melville puffed a bellows +into a tin tube, imitating the lungs, young Alec Bell would +manipulate the "lips," "teeth," and "tongue," causing the thing +to emit high-pitched falsetto gibberish. + +Another would-be technical breakthrough was the Bell "phonautograph" +of 1874, actually made out of a human cadaver's ear. Clamped into place +on a tripod, this grisly gadget drew sound-wave images on smoked glass +through a thin straw glued to its vibrating earbones. + +By 1875, Bell had learned to produce audible sounds--ugly shrieks +and squawks--by using magnets, diaphragms, and electrical current. + +Most "Golden Vaporware" technologies go nowhere. + +But the second stage of technology is the Rising Star, +or, the "Goofy Prototype," stage. The telephone, Bell's +most ambitious gadget yet, reached this stage on March +10, 1876. On that great day, Alexander Graham Bell +became the first person to transmit intelligible human +speech electrically. As it happened, young Professor Bell, +industriously tinkering in his Boston lab, had spattered +his trousers with acid. His assistant, Mr. Watson, +heard his cry for help--over Bell's experimental +audio-telegraph. This was an event without precedent. + +Technologies in their "Goofy Prototype" stage rarely +work very well. They're experimental, and therefore +half- baked and rather frazzled. The prototype may +be attractive and novel, and it does look as if it ought +to be good for something-or-other. But nobody, including +the inventor, is quite sure what. Inventors, and speculators, +and pundits may have very firm ideas about its potential +use, but those ideas are often very wrong. + +The natural habitat of the Goofy Prototype is in trade shows +and in the popular press. Infant technologies need publicity +and investment money like a tottering calf need milk. +This was very true of Bell's machine. To raise research and +development money, Bell toured with his device as a stage attraction. + +Contemporary press reports of the stage debut of the telephone +showed pleased astonishment mixed with considerable dread. +Bell's stage telephone was a large wooden box with a crude +speaker-nozzle, the whole contraption about the size and shape +of an overgrown Brownie camera. Its buzzing steel soundplate, +pumped up by powerful electromagnets, was loud enough to fill +an auditorium. Bell's assistant Mr. Watson, who could manage +on the keyboards fairly well, kicked in by playing the organ +from distant rooms, and, later, distant cities. This feat was +considered marvellous, but very eerie indeed. + +Bell's original notion for the telephone, an idea promoted +for a couple of years, was that it would become a mass medium. +We might recognize Bell's idea today as something close to modern +"cable radio." Telephones at a central source would transmit music, +Sunday sermons, and important public speeches to a paying network +of wired-up subscribers. + +At the time, most people thought this notion made good sense. +In fact, Bell's idea was workable. In Hungary, this philosophy +of the telephone was successfully put into everyday practice. +In Budapest, for decades, from 1893 until after World War I, +there was a government-run information service called +"Telefon Hirmondo-." Hirmondo- was a centralized source +of news and entertainment and culture, including stock reports, +plays, concerts, and novels read aloud. At certain hours +of the day, the phone would ring, you would plug in +a loudspeaker for the use of the family, and Telefon +Hirmondo- would be on the air--or rather, on the phone. + +Hirmondo- is dead tech today, but Hirmondo- might be considered +a spiritual ancestor of the modern telephone-accessed computer +data services, such as CompuServe, GEnie or Prodigy. +The principle behind Hirmondo- is also not too far from computer +"bulletin- board systems" or BBS's, which arrived in the late 1970s, +spread rapidly across America, and will figure largely in this book. + +We are used to using telephones for individual person-to-person speech, +because we are used to the Bell system. But this was just one possibility +among many. Communication networks are very flexible and protean, +especially when their hardware becomes sufficiently advanced. +They can be put to all kinds of uses. And they have been-- +and they will be. + +Bell's telephone was bound for glory, but this was a combination +of political decisions, canny infighting in court, inspired industrial +leadership, receptive local conditions and outright good luck. +Much the same is true of communications systems today. + +As Bell and his backers struggled to install their newfangled system +in the real world of nineteenth-century New England, they had to fight +against skepticism and industrial rivalry. There was already a strong +electrical communications network present in America: the telegraph. +The head of the Western Union telegraph system dismissed Bell's prototype +as "an electrical toy" and refused to buy the rights to Bell's patent. +The telephone, it seemed, might be all right as a parlor entertainment-- +but not for serious business. + +Telegrams, unlike mere telephones, left a permanent physical record +of their messages. Telegrams, unlike telephones, could be answered +whenever the recipient had time and convenience. And the telegram +had a much longer distance-range than Bell's early telephone. +These factors made telegraphy seem a much more sound and businesslike +technology--at least to some. + +The telegraph system was huge, and well-entrenched. +In 1876, the United States had 214,000 miles of telegraph wire, +and 8500 telegraph offices. There were specialized telegraphs +for businesses and stock traders, government, police and fire departments. +And Bell's "toy" was best known as a stage-magic musical device. + +The third stage of technology is known as the "Cash Cow" stage. +In the "cash cow" stage, a technology finds its place in the world, +and matures, and becomes settled and productive. After a year or so, +Alexander Graham Bell and his capitalist backers concluded that +eerie music piped from nineteenth-century cyberspace was not the real +selling-point of his invention. Instead, the telephone was about speech-- +individual, personal speech, the human voice, human conversation and +human interaction. The telephone was not to be managed from any centralized +broadcast center. It was to be a personal, intimate technology. + +When you picked up a telephone, you were not absorbing +the cold output of a machine--you were speaking to another human being. +Once people realized this, their instinctive dread of the telephone +as an eerie, unnatural device, swiftly vanished. A "telephone call" +was not a "call" from a "telephone" itself, but a call from another +human being, someone you would generally know and recognize. +The real point was not what the machine could do for you (or to you), +but what you yourself, a person and citizen, could do THROUGH the machine. +This decision on the part of the young Bell Company was absolutely vital. + +The first telephone networks went up around Boston--mostly among +the technically curious and the well-to-do (much the same segment +of the American populace that, a hundred years later, would be +buying personal computers). Entrenched backers of the telegraph +continued to scoff. + +But in January 1878, a disaster made the telephone famous. +A train crashed in Tarriffville, Connecticut. Forward-looking +doctors in the nearby city of Hartford had had Bell's +"speaking telephone" installed. An alert local druggist +was able to telephone an entire community of local doctors, +who rushed to the site to give aid. The disaster, as disasters do, +aroused intense press coverage. The phone had proven its usefulness +in the real world. + +After Tarriffville, the telephone network spread like crabgrass. +By 1890 it was all over New England. By '93, out to Chicago. +By '97, into Minnesota, Nebraska and Texas. By 1904 it was +all over the continent. + +The telephone had become a mature technology. Professor Bell +(now generally known as "Dr. Bell" despite his lack of a formal degree) +became quite wealthy. He lost interest in the tedious day-to-day business +muddle of the booming telephone network, and gratefully returned +his attention to creatively hacking-around in his various laboratories, +which were now much larger, better-ventilated, and gratifyingly +better-equipped. Bell was never to have another great inventive success, +though his speculations and prototypes anticipated fiber-optic transmission, +manned flight, sonar, hydrofoil ships, tetrahedral construction, and +Montessori education. The "decibel," the standard scientific measure +of sound intensity, was named after Bell. + +Not all Bell's vaporware notions were inspired. He was fascinated +by human eugenics. He also spent many years developing a weird personal +system of astrophysics in which gravity did not exist. + +Bell was a definite eccentric. He was something of a hypochondriac, +and throughout his life he habitually stayed up until four A.M., +refusing to rise before noon. But Bell had accomplished a great feat; +he was an idol of millions and his influence, wealth, and great +personal charm, combined with his eccentricity, made him something +of a loose cannon on deck. Bell maintained a thriving scientific +salon in his winter mansion in Washington, D.C., which gave him +considerable backstage influence in governmental and scientific circles. +He was a major financial backer of the the magazines Science and +National Geographic, both still flourishing today as important organs +of the American scientific establishment. + +Bell's companion Thomas Watson, similarly wealthy and similarly odd, +became the ardent political disciple of a 19th-century science-fiction writer +and would-be social reformer, Edward Bellamy. Watson also trod the boards +briefly as a Shakespearian actor. + +There would never be another Alexander Graham Bell, +but in years to come there would be surprising numbers +of people like him. Bell was a prototype of the +high-tech entrepreneur. High-tech entrepreneurs will +play a very prominent role in this book: not merely as +technicians and businessmen, but as pioneers of the +technical frontier, who can carry the power and prestige +they derive from high-technology into the political and +social arena. + +Like later entrepreneurs, Bell was fierce in defense of +his own technological territory. As the telephone began to +flourish, Bell was soon involved in violent lawsuits in the +defense of his patents. Bell's Boston lawyers were +excellent, however, and Bell himself, as an elocution +teacher and gifted public speaker, was a devastatingly +effective legal witness. In the eighteen years of Bell's patents, +the Bell company was involved in six hundred separate lawsuits. +The legal records printed filled 149 volumes. The Bell Company +won every single suit. + +After Bell's exclusive patents expired, rival telephone +companies sprang up all over America. Bell's company, +American Bell Telephone, was soon in deep trouble. +In 1907, American Bell Telephone fell into the hands of the +rather sinister J.P. Morgan financial cartel, robber-baron +speculators who dominated Wall Street. + +At this point, history might have taken a different turn. +American might well have been served forever by a patchwork +of locally owned telephone companies. Many state politicians +and local businessmen considered this an excellent solution. + +But the new Bell holding company, American Telephone and Telegraph +or AT&T, put in a new man at the helm, a visionary industrialist +named Theodore Vail. Vail, a former Post Office manager, +understood large organizations and had an innate feeling +for the nature of large-scale communications. Vail quickly +saw to it that AT&T seized the technological edge once again. +The Pupin and Campbell "loading coil," and the deForest +"audion," are both extinct technology today, but in 1913 +they gave Vail's company the best LONG-DISTANCE lines +ever built. By controlling long-distance--the links +between, and over, and above the smaller local phone +companies--AT&T swiftly gained the whip-hand over them, +and was soon devouring them right and left. + +Vail plowed the profits back into research and development, +starting the Bell tradition of huge-scale and brilliant +industrial research. + +Technically and financially, AT&T gradually steamrollered +the opposition. Independent telephone companies never +became entirely extinct, and hundreds of them flourish today. +But Vail's AT&T became the supreme communications company. +At one point, Vail's AT&T bought Western Union itself, +the very company that had derided Bell's telephone as a "toy." +Vail thoroughly reformed Western Union's hidebound business +along his modern principles; but when the federal government +grew anxious at this centralization of power, Vail politely +gave Western Union back. + +This centralizing process was not unique. Very similar +events had happened in American steel, oil, and railroads. +But AT&T, unlike the other companies, was to remain supreme. +The monopoly robber-barons of those other industries +were humbled and shattered by government trust-busting. + +Vail, the former Post Office official, was quite willing +to accommodate the US government; in fact he would +forge an active alliance with it. AT&T would become +almost a wing of the American government, almost +another Post Office--though not quite. AT&T would +willingly submit to federal regulation, but in return, +it would use the government's regulators as its own police, +who would keep out competitors and assure the Bell +system's profits and preeminence. + +This was the second birth--the political birth--of the +American telephone system. Vail's arrangement was to +persist, with vast success, for many decades, until 1982. +His system was an odd kind of American industrial socialism. +It was born at about the same time as Leninist Communism, +and it lasted almost as long--and, it must be admitted, +to considerably better effect. + +Vail's system worked. Except perhaps for aerospace, +there has been no technology more thoroughly dominated +by Americans than the telephone. The telephone was +seen from the beginning as a quintessentially American +technology. Bell's policy, and the policy of Theodore Vail, +was a profoundly democratic policy of UNIVERSAL ACCESS. +Vail's famous corporate slogan, "One Policy, One System, +Universal Service," was a political slogan, with a very +American ring to it. + +The American telephone was not to become the specialized tool +of government or business, but a general public utility. +At first, it was true, only the wealthy could afford +private telephones, and Bell's company pursued the +business markets primarily. The American phone system +was a capitalist effort, meant to make money; it was not a charity. +But from the first, almost all communities with telephone service +had public telephones. And many stores--especially drugstores-- +offered public use of their phones. You might not own a telephone-- +but you could always get into the system, if you really needed to. + +There was nothing inevitable about this decision to make telephones +"public" and "universal." Vail's system involved a profound act +of trust in the public. This decision was a political one, +informed by the basic values of the American republic. +The situation might have been very different; +and in other countries, under other systems, +it certainly was. + +Joseph Stalin, for instance, vetoed plans for a Soviet +phone system soon after the Bolshevik revolution. +Stalin was certain that publicly accessible telephones +would become instruments of anti-Soviet counterrevolution +and conspiracy. (He was probably right.) When telephones +did arrive in the Soviet Union, they would be instruments +of Party authority, and always heavily tapped. (Alexander +Solzhenitsyn's prison-camp novel The First Circle +describes efforts to develop a phone system more suited +to Stalinist purposes.) + +France, with its tradition of rational centralized government, +had fought bitterly even against the electric telegraph, +which seemed to the French entirely too anarchical and frivolous. +For decades, nineteenth-century France communicated via the +"visual telegraph," a nation-spanning, government-owned semaphore +system of huge stone towers that signalled from hilltops, +across vast distances, with big windmill-like arms. +In 1846, one Dr. Barbay, a semaphore enthusiast, +memorably uttered an early version of what might be called +"the security expert's argument" against the open media. + +"No, the electric telegraph is not a sound invention. +It will always be at the mercy of the slightest disruption, +wild youths, drunkards, bums, etc. . . . The electric telegraph +meets those destructive elements with only a few meters of wire +over which supervision is impossible. A single man could, +without being seen, cut the telegraph wires leading to Paris, +and in twenty-four hours cut in ten different places the wires +of the same line, without being arrested. The visual telegraph, +on the contrary, has its towers, its high walls, its gates +well-guarded from inside by strong armed men. Yes, I declare, +substitution of the electric telegraph for the visual one +is a dreadful measure, a truly idiotic act." + +Dr. Barbay and his high-security stone machines +were eventually unsuccessful, but his argument-- +that communication exists for the safety and convenience +of the state, and must be carefully protected from the wild +boys and the gutter rabble who might want to crash the +system--would be heard again and again. + +When the French telephone system finally did arrive, +its snarled inadequacy was to be notorious. Devotees +of the American Bell System often recommended a trip +to France, for skeptics. + +In Edwardian Britain, issues of class and privacy +were a ball-and-chain for telephonic progress. It was +considered outrageous that anyone--any wild fool off +the street--could simply barge bellowing into one's office +or home, preceded only by the ringing of a telephone bell. +In Britain, phones were tolerated for the use of business, +but private phones tended be stuffed away into closets, +smoking rooms, or servants' quarters. Telephone operators +were resented in Britain because they did not seem to +"know their place." And no one of breeding would print +a telephone number on a business card; this seemed a crass +attempt to make the acquaintance of strangers. + +But phone access in America was to become a popular right; +something like universal suffrage, only more so. +American women could not yet vote when the phone system +came through; yet from the beginning American women +doted on the telephone. This "feminization" of the +American telephone was often commented on by foreigners. +Phones in America were not censored or stiff or formalized; +they were social, private, intimate, and domestic. +In America, Mother's Day is by far the busiest day +of the year for the phone network. + +The early telephone companies, and especially AT&T, +were among the foremost employers of American women. +They employed the daughters of the American middle-class +in great armies: in 1891, eight thousand women; by 1946, +almost a quarter of a million. Women seemed to enjoy +telephone work; it was respectable, it was steady, +it paid fairly well as women's work went, and--not least-- +it seemed a genuine contribution to the social good +of the community. Women found Vail's ideal of public +service attractive. This was especially true in rural areas, +where women operators, running extensive rural party-lines, +enjoyed considerable social power. The operator knew everyone +on the party-line, and everyone knew her. + +Although Bell himself was an ardent suffragist, the +telephone company did not employ women for the sake of +advancing female liberation. AT&T did this for sound +commercial reasons. The first telephone operators of +the Bell system were not women, but teenage American boys. +They were telegraphic messenger boys (a group about to +be rendered technically obsolescent), who swept up +around the phone office, dunned customers for bills, +and made phone connections on the switchboard, +all on the cheap. + +Within the very first year of operation, 1878, +Bell's company learned a sharp lesson about combining +teenage boys and telephone switchboards. Putting +teenage boys in charge of the phone system brought swift +and consistent disaster. Bell's chief engineer described them +as "Wild Indians." The boys were openly rude to customers. +They talked back to subscribers, saucing off, +uttering facetious remarks, and generally giving lip. +The rascals took Saint Patrick's Day off without permission. +And worst of all they played clever tricks with +the switchboard plugs: disconnecting calls, crossing lines +so that customers found themselves talking to strangers, +and so forth. + +This combination of power, technical mastery, and effective +anonymity seemed to act like catnip on teenage boys. + +This wild-kid-on-the-wires phenomenon was not confined to +the USA; from the beginning, the same was true of the British +phone system. An early British commentator kindly remarked: +"No doubt boys in their teens found the work not a little irksome, +and it is also highly probable that under the early conditions +of employment the adventurous and inquisitive spirits of which +the average healthy boy of that age is possessed, were not always +conducive to the best attention being given to the wants +of the telephone subscribers." + +So the boys were flung off the system--or at least, +deprived of control of the switchboard. But the +"adventurous and inquisitive spirits" of the teenage boys +would be heard from in the world of telephony, again and again. + +The fourth stage in the technological life-cycle is death: +"the Dog," dead tech. The telephone has so far avoided this fate. +On the contrary, it is thriving, still spreading, still evolving, +and at increasing speed. + +The telephone has achieved a rare and exalted state for a +technological artifact: it has become a HOUSEHOLD OBJECT. +The telephone, like the clock, like pen and paper, +like kitchen utensils and running water, has become +a technology that is visible only by its absence. +The telephone is technologically transparent. +The global telephone system is the largest and most +complex machine in the world, yet it is easy to use. +More remarkable yet, the telephone is almost entirely +physically safe for the user. + +For the average citizen in the 1870s, the telephone +was weirder, more shocking, more "high-tech" and +harder to comprehend, than the most outrageous stunts +of advanced computing for us Americans in the 1990s. +In trying to understand what is happening to us today, +with our bulletin-board systems, direct overseas dialling, +fiber-optic transmissions, computer viruses, hacking stunts, +and a vivid tangle of new laws and new crimes, it is important +to realize that our society has been through a similar challenge before-- +and that, all in all, we did rather well by it. + +Bell's stage telephone seemed bizarre at first. But the +sensations of weirdness vanished quickly, once people began +to hear the familiar voices of relatives and friends, +in their own homes on their own telephones. The telephone +changed from a fearsome high-tech totem to an everyday pillar +of human community. + +This has also happened, and is still happening, +to computer networks. Computer networks such as +NSFnet, BITnet, USENET, JANET, are technically +advanced, intimidating, and much harder to use than +telephones. Even the popular, commercial computer +networks, such as GEnie, Prodigy, and CompuServe, +cause much head-scratching and have been described +as "user-hateful." Nevertheless they too are changing +from fancy high-tech items into everyday sources +of human community. + +The words "community" and "communication" have +the same root. Wherever you put a communications +network, you put a community as well. And whenever +you TAKE AWAY that network--confiscate it, outlaw it, +crash it, raise its price beyond affordability-- +then you hurt that community. + +Communities will fight to defend themselves. People will fight harder +and more bitterly to defend their communities, than they will fight +to defend their own individual selves. And this is very true +of the "electronic community" that arose around computer networks +in the 1980s--or rather, the VARIOUS electronic communities, +in telephony, law enforcement, computing, and the digital +underground that, by the year 1990, were raiding, rallying, +arresting, suing, jailing, fining and issuing angry manifestos. + +None of the events of 1990 were entirely new. +Nothing happened in 1990 that did not have some kind +of earlier and more understandable precedent. What gave +the Hacker Crackdown its new sense of gravity and +importance was the feeling--the COMMUNITY feeling-- +that the political stakes had been raised; that trouble +in cyberspace was no longer mere mischief or inconclusive +skirmishing, but a genuine fight over genuine issues, +a fight for community survival and the shape of the future. + +These electronic communities, having flourished throughout +the 1980s, were becoming aware of themselves, and increasingly, +becoming aware of other, rival communities. Worries were +sprouting up right and left, with complaints, rumors, +uneasy speculations. But it would take a catalyst, a shock, +to make the new world evident. Like Bell's great publicity break, +the Tarriffville Rail Disaster of January 1878, +it would take a cause celebre. + +That cause was the AT&T Crash of January 15, 1990. +After the Crash, the wounded and anxious telephone +community would come out fighting hard. + +# + +The community of telephone technicians, engineers, operators +and researchers is the oldest community in cyberspace. +These are the veterans, the most developed group, +the richest, the most respectable, in most ways the most powerful. +Whole generations have come and gone since Alexander Graham Bell's day, +but the community he founded survives; people work for the phone system +today whose great-grandparents worked for the phone system. +Its specialty magazines, such as Telephony, AT&T Technical Journal, +Telephone Engineer and Management, are decades old; +they make computer publications like Macworld and PC Week +look like amateur johnny-come-latelies. + +And the phone companies take no back seat in high-technology, either. +Other companies' industrial researchers may have won new markets; +but the researchers of Bell Labs have won SEVEN NOBEL PRIZES. +One potent device that Bell Labs originated, the transistor, +has created entire GROUPS of industries. Bell Labs are +world-famous for generating "a patent a day," and have even +made vital discoveries in astronomy, physics and cosmology. + +Throughout its seventy-year history, "Ma Bell" was not so much +a company as a way of life. Until the cataclysmic divestiture +of the 1980s, Ma Bell was perhaps the ultimate maternalist mega-employer. +The AT&T corporate image was the "gentle giant," "the voice with a smile," +a vaguely socialist-realist world of cleanshaven linemen in shiny helmets +and blandly pretty phone-girls in headsets and nylons. Bell System +employees were famous as rock-ribbed Kiwanis and Rotary members, +Little-League enthusiasts, school-board people. + +During the long heyday of Ma Bell, the Bell employee corps +were nurtured top-to-bottom on a corporate ethos of public service. +There was good money in Bell, but Bell was not ABOUT money; +Bell used public relations, but never mere marketeering. +People went into the Bell System for a good life, +and they had a good life. But it was not mere money +that led Bell people out in the midst of storms and earthquakes +to fight with toppled phone-poles, to wade in flooded manholes, +to pull the red-eyed graveyard-shift over collapsing switching-systems. +The Bell ethic was the electrical equivalent of the postman's: +neither rain, nor snow, nor gloom of night would stop these couriers. + +It is easy to be cynical about this, as it is easy to be +cynical about any political or social system; but cynicism +does not change the fact that thousands of people took +these ideals very seriously. And some still do. + +The Bell ethos was about public service; and that was +gratifying; but it was also about private POWER, and that +was gratifying too. As a corporation, Bell was very special. +Bell was privileged. Bell had snuggled up close to the state. +In fact, Bell was as close to government as you could get in +America and still make a whole lot of legitimate money. + +But unlike other companies, Bell was above and beyond +the vulgar commercial fray. Through its regional operating companies, +Bell was omnipresent, local, and intimate, all over America; +but the central ivory towers at its corporate heart were the +tallest and the ivoriest around. + +There were other phone companies in America, to be sure; +the so-called independents. Rural cooperatives, mostly; +small fry, mostly tolerated, sometimes warred upon. +For many decades, "independent" American phone companies +lived in fear and loathing of the official Bell monopoly +(or the "Bell Octopus," as Ma Bell's nineteenth-century +enemies described her in many angry newspaper manifestos). +Some few of these independent entrepreneurs, while legally +in the wrong, fought so bitterly against the Octopus +that their illegal phone networks were cast into the street +by Bell agents and publicly burned. + +The pure technical sweetness of the Bell System gave its operators, +inventors and engineers a deeply satisfying sense of power and mastery. +They had devoted their lives to improving this vast nation-spanning machine; +over years, whole human lives, they had watched it improve and grow. +It was like a great technological temple. They were an elite, +and they knew it--even if others did not; in fact, they felt +even more powerful BECAUSE others did not understand. + +The deep attraction of this sensation of elite technical power +should never be underestimated. "Technical power" is not for everybody; +for many people it simply has no charm at all. But for some people, +it becomes the core of their lives. For a few, it is overwhelming, +obsessive; it becomes something close to an addiction. People--especially +clever teenage boys whose lives are otherwise mostly powerless and put-upon +--love this sensation of secret power, and are willing to do all sorts +of amazing things to achieve it. The technical POWER of electronics +has motivated many strange acts detailed in this book, which would +otherwise be inexplicable. + +So Bell had power beyond mere capitalism. The Bell service ethos worked, +and was often propagandized, in a rather saccharine fashion. Over the decades, +people slowly grew tired of this. And then, openly impatient with it. +By the early 1980s, Ma Bell was to find herself with scarcely a real friend +in the world. Vail's industrial socialism had become hopelessly +out-of-fashion politically. Bell would be punished for that. +And that punishment would fall harshly upon the people of the +telephone community. + +# + +In 1983, Ma Bell was dismantled by federal court action. +The pieces of Bell are now separate corporate entities. +The core of the company became AT&T Communications, +and also AT&T Industries (formerly Western Electric, +Bell's manufacturing arm). AT&T Bell Labs became Bell +Communications Research, Bellcore. Then there are the +Regional Bell Operating Companies, or RBOCs, pronounced "arbocks." + +Bell was a titan and even these regional chunks are gigantic enterprises: +Fortune 50 companies with plenty of wealth and power behind them. +But the clean lines of "One Policy, One System, Universal Service" +have been shattered, apparently forever. + +The "One Policy" of the early Reagan Administration was to +shatter a system that smacked of noncompetitive socialism. +Since that time, there has been no real telephone "policy" +on the federal level. Despite the breakup, the remnants +of Bell have never been set free to compete in the open marketplace. + +The RBOCs are still very heavily regulated, but not from the top. +Instead, they struggle politically, economically and legally, +in what seems an endless turmoil, in a patchwork of overlapping federal +and state jurisdictions. Increasingly, like other major American corporations, +the RBOCs are becoming multinational, acquiring important commercial interests +in Europe, Latin America, and the Pacific Rim. But this, too, adds to their +legal and political predicament. + +The people of what used to be Ma Bell are not happy about their fate. +They feel ill-used. They might have been grudgingly willing to make +a full transition to the free market; to become just companies amid +other companies. But this never happened. Instead, AT&T and the RBOCS +("the Baby Bells") feel themselves wrenched from side to side by state +regulators, by Congress, by the FCC, and especially by the federal court +of Judge Harold Greene, the magistrate who ordered the Bell breakup +and who has been the de facto czar of American telecommunications +ever since 1983. + +Bell people feel that they exist in a kind of paralegal limbo today. +They don't understand what's demanded of them. If it's "service," +why aren't they treated like a public service? And if it's money, +then why aren't they free to compete for it? No one seems to know, +really. Those who claim to know keep changing their minds. +Nobody in authority seems willing to grasp the nettle for once and all. + +Telephone people from other countries are amazed by the +American telephone system today. Not that it works so well; +for nowadays even the French telephone system works, more or less. +They are amazed that the American telephone system STILL works +AT ALL, under these strange conditions. + +Bell's "One System" of long-distance service is now only about +eighty percent of a system, with the remainder held by Sprint, MCI, +and the midget long-distance companies. Ugly wars over dubious +corporate practices such as "slamming" (an underhanded method +of snitching clients from rivals) break out with some regularity +in the realm of long-distance service. The battle to break Bell's +long-distance monopoly was long and ugly, and since the breakup +the battlefield has not become much prettier. AT&T's famous +shame-and-blame advertisements, which emphasized the shoddy work +and purported ethical shadiness of their competitors, were much +remarked on for their studied psychological cruelty. + +There is much bad blood in this industry, and much +long-treasured resentment. AT&T's post-breakup +corporate logo, a striped sphere, is known in the +industry as the "Death Star" (a reference from the movie +Star Wars, in which the "Death Star" was the spherical +high- tech fortress of the harsh-breathing imperial ultra-baddie, +Darth Vader.) Even AT&T employees are less than thrilled +by the Death Star. A popular (though banned) T-shirt among +AT&T employees bears the old-fashioned Bell logo of the Bell System, +plus the newfangled striped sphere, with the before-and-after comments: +"This is your brain--This is your brain on drugs!" AT&T made a very +well-financed and determined effort to break into the personal +computer market; it was disastrous, and telco computer experts +are derisively known by their competitors as "the pole-climbers." +AT&T and the Baby Bell arbocks still seem to have few friends. + +Under conditions of sharp commercial competition, a crash like +that of January 15, 1990 was a major embarrassment to AT&T. +It was a direct blow against their much-treasured reputation +for reliability. Within days of the crash AT&T's +Chief Executive Officer, Bob Allen, officially apologized, +in terms of deeply pained humility: + +"AT&T had a major service disruption last Monday. +We didn't live up to our own standards of quality, +and we didn't live up to yours. It's as simple as that. +And that's not acceptable to us. Or to you. . . . +We understand how much people have come to depend +upon AT&T service, so our AT&T Bell Laboratories scientists +and our network engineers are doing everything possible +to guard against a recurrence. . . . We know there's no way +to make up for the inconvenience this problem may have caused you." + +Mr Allen's "open letter to customers" was printed in lavish ads +all over the country: in the Wall Street Journal, USA Today, +New York Times, Los Angeles Times, Chicago Tribune, +Philadelphia Inquirer, San Francisco Chronicle Examiner, +Boston Globe, Dallas Morning News, Detroit Free Press, +Washington Post, Houston Chronicle, Cleveland Plain Dealer, +Atlanta Journal Constitution, Minneapolis Star Tribune, +St. Paul Pioneer Press Dispatch, Seattle Times/Post Intelligencer, +Tacoma News Tribune, Miami Herald, Pittsburgh Press, +St. Louis Post Dispatch, Denver Post, Phoenix Republic Gazette +and Tampa Tribune. + +In another press release, AT&T went to some pains to suggest +that this "software glitch" might have happened just as easily to MCI, +although, in fact, it hadn't. (MCI's switching software was quite different +from AT&T's--though not necessarily any safer.) AT&T also announced +their plans to offer a rebate of service on Valentine's Day to make up +for the loss during the Crash. + +"Every technical resource available, including Bell Labs +scientists and engineers, has been devoted to assuring +it will not occur again," the public was told. They were +further assured that "The chances of a recurrence are small-- +a problem of this magnitude never occurred before." + +In the meantime, however, police and corporate +security maintained their own suspicions about +"the chances of recurrence" and the real reason why +a "problem of this magnitude" had appeared, seemingly +out of nowhere. Police and security knew for a fact +that hackers of unprecedented sophistication were illegally +entering, and reprogramming, certain digital switching stations. +Rumors of hidden "viruses" and secret "logic bombs" +in the switches ran rampant in the underground, +with much chortling over AT&T's predicament, +and idle speculation over what unsung hacker genius +was responsible for it. Some hackers, including police +informants, were trying hard to finger one another +as the true culprits of the Crash. + +Telco people found little comfort in objectivity when +they contemplated these possibilities. It was just too close +to the bone for them; it was embarrassing; it hurt so much, +it was hard even to talk about. + +There has always been thieving and misbehavior in the phone system. +There has always been trouble with the rival independents, +and in the local loops. But to have such trouble in the core +of the system, the long-distance switching stations, +is a horrifying affair. To telco people, this is +all the difference between finding roaches in your kitchen +and big horrid sewer-rats in your bedroom. + +From the outside, to the average citizen, the telcos +still seem gigantic and impersonal. The American public +seems to regard them as something akin to Soviet apparats. +Even when the telcos do their best corporate-citizen routine, +subsidizing magnet high-schools and sponsoring news-shows +on public television, they seem to win little except public suspicion. + +But from the inside, all this looks very different. +There's harsh competition. A legal and political system +that seems baffled and bored, when not actively hostile +to telco interests. There's a loss of morale, a deep sensation +of having somehow lost the upper hand. Technological change +has caused a loss of data and revenue to other, newer forms +of transmission. There's theft, and new forms of theft, +of growing scale and boldness and sophistication. +With all these factors, it was no surprise to see the telcos, +large and small, break out in a litany of bitter complaint. + +In late '88 and throughout 1989, telco representatives +grew shrill in their complaints to those few American law +enforcement officials who make it their business to try to +understand what telephone people are talking about. +Telco security officials had discovered the computer- +hacker underground, infiltrated it thoroughly, +and become deeply alarmed at its growing expertise. +Here they had found a target that was not only loathsome +on its face, but clearly ripe for counterattack. + +Those bitter rivals: AT&T, MCI and Sprint--and a crowd +of Baby Bells: PacBell, Bell South, Southwestern Bell, +NYNEX, USWest, as well as the Bell research consortium Bellcore, +and the independent long-distance carrier Mid-American-- +all were to have their role in the great hacker dragnet of 1990. +After years of being battered and pushed around, the telcos had, +at least in a small way, seized the initiative again. +After years of turmoil, telcos and government officials were +once again to work smoothly in concert in defense of the System. +Optimism blossomed; enthusiasm grew on all sides; +the prospective taste of vengeance was sweet. + +# + +From the beginning--even before the crackdown had a name-- +secrecy was a big problem. There were many good reasons +for secrecy in the hacker crackdown. Hackers and code-thieves +were wily prey, slinking back to their bedrooms and basements +and destroying vital incriminating evidence at the first hint of trouble. +Furthermore, the crimes themselves were heavily technical and difficult +to describe, even to police--much less to the general public. + +When such crimes HAD been described intelligibly to the public, +in the past, that very publicity had tended to INCREASE the crimes +enormously. Telco officials, while painfully aware of the vulnerabilities +of their systems, were anxious not to publicize those weaknesses. +Experience showed them that those weaknesses, once discovered, +would be pitilessly exploited by tens of thousands of people--not only +by professional grifters and by underground hackers and phone phreaks, +but by many otherwise more-or-less honest everyday folks, who regarded +stealing service from the faceless, soulless "Phone Company" as a kind of +harmless indoor sport. When it came to protecting their interests, +telcos had long since given up on general public sympathy for +"the Voice with a Smile." Nowadays the telco's "Voice" was +very likely to be a computer's; and the American public +showed much less of the proper respect and gratitude due +the fine public service bequeathed them by Dr. Bell and Mr. Vail. +The more efficient, high-tech, computerized, and impersonal +the telcos became, it seemed, the more they were met by +sullen public resentment and amoral greed. + +Telco officials wanted to punish the phone-phreak underground, in as +public and exemplary a manner as possible. They wanted to make dire +examples of the worst offenders, to seize the ringleaders and intimidate +the small fry, to discourage and frighten the wacky hobbyists, and send +the professional grifters to jail. To do all this, publicity was vital. + +Yet operational secrecy was even more so. If word got out that +a nationwide crackdown was coming, the hackers might simply vanish; +destroy the evidence, hide their computers, go to earth, +and wait for the campaign to blow over. Even the young +hackers were crafty and suspicious, and as for the professional grifters, +they tended to split for the nearest state-line at the first sign of trouble. +For the crackdown to work well, they would all have to be caught red-handed, +swept upon suddenly, out of the blue, from every corner of the compass. + +And there was another strong motive for secrecy. In the worst-case scenario, +a blown campaign might leave the telcos open to a devastating hacker +counter-attack. If there were indeed hackers loose in America who +had caused the January 15 Crash--if there were truly gifted hackers, +loose in the nation's long-distance switching systems, and enraged +or frightened by the crackdown--then they might react unpredictably +to an attempt to collar them. Even if caught, they might have talented +and vengeful friends still running around loose. Conceivably, +it could turn ugly. Very ugly. In fact, it was hard to imagine +just how ugly things might turn, given that possibility. + +Counter-attack from hackers was a genuine concern for the telcos. +In point of fact, they would never suffer any such counter-attack. +But in months to come, they would be at some pains to publicize +this notion and to utter grim warnings about it. + +Still, that risk seemed well worth running. Better to run the risk +of vengeful attacks, than to live at the mercy of potential crashers. +Any cop would tell you that a protection racket had no real future. + +And publicity was such a useful thing. Corporate security officers, +including telco security, generally work under conditions of great discretion. +And corporate security officials do not make money for their companies. +Their job is to PREVENT THE LOSS of money, which is much less glamorous +than actually winning profits. + +If you are a corporate security official, and you do your job brilliantly, +then nothing bad happens to your company at all. Because of this, you appear +completely superfluous. This is one of the many unattractive aspects +of security work. It's rare that these folks have the chance to draw +some healthy attention to their own efforts. + +Publicity also served the interest of their friends in law enforcement. +Public officials, including law enforcement officials, thrive by attracting +favorable public interest. A brilliant prosecution in a matter of vital +public interest can make the career of a prosecuting attorney. +And for a police officer, good publicity opens the purses of the legislature; +it may bring a citation, or a promotion, or at least a rise in status +and the respect of one's peers. + +But to have both publicity and secrecy is to have one's cake and eat it too. +In months to come, as we will show, this impossible act was to cause great +pain to the agents of the crackdown. But early on, it seemed possible +--maybe even likely--that the crackdown could successfully combine +the best of both worlds. The ARREST of hackers would be heavily publicized. +The actual DEEDS of the hackers, which were technically hard to explain +and also a security risk, would be left decently obscured. The THREAT +hackers posed would be heavily trumpeted; the likelihood of their actually +committing such fearsome crimes would be left to the public's imagination. +The spread of the computer underground, and its growing technical +sophistication, would be heavily promoted; the actual hackers themselves, +mostly bespectacled middle-class white suburban teenagers, +would be denied any personal publicity. + +It does not seem to have occurred to any telco official +that the hackers accused would demand a day in court; +that journalists would smile upon the hackers as +"good copy;" that wealthy high-tech entrepreneurs would +offer moral and financial support to crackdown victims; +that constitutional lawyers would show up with briefcases, +frowning mightily. This possibility does not seem to have +ever entered the game-plan. + +And even if it had, it probably would not have slowed +the ferocious pursuit of a stolen phone-company document, +mellifluously known as "Control Office Administration of +Enhanced 911 Services for Special Services and Major Account Centers." + +In the chapters to follow, we will explore the worlds +of police and the computer underground, and the large +shadowy area where they overlap. But first, we must +explore the battleground. Before we leave the world +of the telcos, we must understand what a switching system +actually is and how your telephone actually works. + +# + +To the average citizen, the idea of the telephone is represented by, +well, a TELEPHONE: a device that you talk into. To a telco +professional, however, the telephone itself is known, in lordly +fashion, as a "subset." The "subset" in your house is a mere adjunct, +a distant nerve ending, of the central switching stations, +which are ranked in levels of heirarchy, up to the long-distance electronic +switching stations, which are some of the largest computers on earth. + +Let us imagine that it is, say, 1925, before the +introduction of computers, when the phone system was +simpler and somewhat easier to grasp. Let's further +imagine that you are Miss Leticia Luthor, a fictional +operator for Ma Bell in New York City of the 20s. + +Basically, you, Miss Luthor, ARE the "switching system." +You are sitting in front of a large vertical switchboard, +known as a "cordboard," made of shiny wooden panels, +with ten thousand metal-rimmed holes punched in them, +known as jacks. The engineers would have put more +holes into your switchboard, but ten thousand is +as many as you can reach without actually having +to get up out of your chair. + +Each of these ten thousand holes has its own little electric lightbulb, +known as a "lamp," and its own neatly printed number code. + +With the ease of long habit, you are scanning your board for lit-up bulbs. +This is what you do most of the time, so you are used to it. + +A lamp lights up. This means that the phone +at the end of that line has been taken off the hook. +Whenever a handset is taken off the hook, that closes a circuit +inside the phone which then signals the local office, i.e. you, +automatically. There might be somebody calling, or then +again the phone might be simply off the hook, but this +does not matter to you yet. The first thing you do, +is record that number in your logbook, in your fine American +public-school handwriting. This comes first, naturally, +since it is done for billing purposes. + +You now take the plug of your answering cord, which goes +directly to your headset, and plug it into the lit-up hole. +"Operator," you announce. + +In operator's classes, before taking this job, you have +been issued a large pamphlet full of canned operator's +responses for all kinds of contingencies, which you had +to memorize. You have also been trained in a proper +non-regional, non-ethnic pronunciation and tone of voice. +You rarely have the occasion to make any spontaneous +remark to a customer, and in fact this is frowned upon +(except out on the rural lines where people have time +on their hands and get up to all kinds of mischief). + +A tough-sounding user's voice at the end of the line +gives you a number. Immediately, you write that number +down in your logbook, next to the caller's number, +which you just wrote earlier. You then look and see if +the number this guy wants is in fact on your switchboard, +which it generally is, since it's generally a local call. +Long distance costs so much that people use it sparingly. + +Only then do you pick up a calling-cord from a shelf +at the base of the switchboard. This is a long elastic cord +mounted on a kind of reel so that it will zip back in when +you unplug it. There are a lot of cords down there, +and when a bunch of them are out at once they look like +a nest of snakes. Some of the girls think there are bugs +living in those cable-holes. They're called "cable mites" +and are supposed to bite your hands and give you rashes. +You don't believe this, yourself. + +Gripping the head of your calling-cord, you slip the tip +of it deftly into the sleeve of the jack for the called person. +Not all the way in, though. You just touch it. If you hear +a clicking sound, that means the line is busy and you can't +put the call through. If the line is busy, you have to stick +the calling-cord into a "busy-tone jack," which will give +the guy a busy-tone. This way you don't have to talk to him +yourself and absorb his natural human frustration. + +But the line isn't busy. So you pop the cord all the way in. +Relay circuits in your board make the distant phone ring, +and if somebody picks it up off the hook, then a phone +conversation starts. You can hear this conversation +on your answering cord, until you unplug it. In fact +you could listen to the whole conversation if you wanted, +but this is sternly frowned upon by management, and frankly, +when you've overheard one, you've pretty much heard 'em all. + +You can tell how long the conversation lasts by the glow +of the calling-cord's lamp, down on the calling-cord's shelf. +When it's over, you unplug and the calling-cord zips back into place. + +Having done this stuff a few hundred thousand times, +you become quite good at it. In fact you're plugging, +and connecting, and disconnecting, ten, twenty, forty cords +at a time. It's a manual handicraft, really, quite satisfying +in a way, rather like weaving on an upright loom. + +Should a long-distance call come up, it would be different, +but not all that different. Instead of connecting the call +through your own local switchboard, you have to go up the hierarchy, +onto the long-distance lines, known as "trunklines." +Depending on how far the call goes, it may have to work +its way through a whole series of operators, which can +take quite a while. The caller doesn't wait on the line +while this complex process is negotiated across the country +by the gaggle of operators. Instead, the caller hangs up, +and you call him back yourself when the call has finally +worked its way through. + +After four or five years of this work, you get married, +and you have to quit your job, this being the natural order +of womanhood in the American 1920s. The phone company +has to train somebody else--maybe two people, since +the phone system has grown somewhat in the meantime. +And this costs money. + +In fact, to use any kind of human being as a switching +system is a very expensive proposition. Eight thousand +Leticia Luthors would be bad enough, but a quarter of a +million of them is a military-scale proposition and makes +drastic measures in automation financially worthwhile. + +Although the phone system continues to grow today, +the number of human beings employed by telcos has +been dropping steadily for years. Phone "operators" +now deal with nothing but unusual contingencies, +all routine operations having been shrugged off onto machines. +Consequently, telephone operators are considerably less +machine-like nowadays, and have been known to have accents +and actual character in their voices. When you reach +a human operator today, the operators are rather more +"human" than they were in Leticia's day--but on the other hand, +human beings in the phone system are much harder to reach +in the first place. + +Over the first half of the twentieth century, +"electromechanical" switching systems of growing +complexity were cautiously introduced into the phone system. +In certain backwaters, some of these hybrid systems are still +in use. But after 1965, the phone system began to go completely +electronic, and this is by far the dominant mode today. +Electromechanical systems have "crossbars," and "brushes," +and other large moving mechanical parts, which, while faster +and cheaper than Leticia, are still slow, and tend to wear out +fairly quickly. + +But fully electronic systems are inscribed on silicon chips, +and are lightning-fast, very cheap, and quite durable. +They are much cheaper to maintain than even the best +electromechanical systems, and they fit into half the space. +And with every year, the silicon chip grows smaller, faster, +and cheaper yet. Best of all, automated electronics work +around the clock and don't have salaries or health insurance. + +There are, however, quite serious drawbacks to the +use of computer-chips. When they do break down, it is +a daunting challenge to figure out what the heck has gone +wrong with them. A broken cordboard generally had +a problem in it big enough to see. A broken chip has +invisible, microscopic faults. And the faults in bad +software can be so subtle as to be practically theological. + +If you want a mechanical system to do something new, +then you must travel to where it is, and pull pieces out of it, +and wire in new pieces. This costs money. However, if you want +a chip to do something new, all you have to do is change its software, +which is easy, fast and dirt-cheap. You don't even have to see the chip +to change its program. Even if you did see the chip, it wouldn't look +like much. A chip with program X doesn't look one whit different from +a chip with program Y. + +With the proper codes and sequences, and access to specialized phone-lines, +you can change electronic switching systems all over America from anywhere +you please. + +And so can other people. If they know how, and if they want to, +they can sneak into a microchip via the special phonelines and diddle with it, +leaving no physical trace at all. If they broke into the operator's station +and held Leticia at gunpoint, that would be very obvious. If they broke into +a telco building and went after an electromechanical switch with a toolbelt, +that would at least leave many traces. But people can do all manner of amazing +things to computer switches just by typing on a keyboard, and keyboards are +everywhere today. The extent of this vulnerability is deep, dark, broad, +almost mind-boggling, and yet this is a basic, primal fact of life about +any computer on a network. + +Security experts over the past twenty years have insisted, +with growing urgency, that this basic vulnerability of computers +represents an entirely new level of risk, of unknown but obviously +dire potential to society. And they are right. + +An electronic switching station does pretty much +everything Letitia did, except in nanoseconds and +on a much larger scale. Compared to Miss Luthor's +ten thousand jacks, even a primitive 1ESS switching computer, +60s vintage, has a 128,000 lines. And the current AT&T +system of choice is the monstrous fifth-generation 5ESS. + +An Electronic Switching Station can scan every line on its "board" +in a tenth of a second, and it does this over and over, tirelessly, +around the clock. Instead of eyes, it uses "ferrod scanners" +to check the condition of local lines and trunks. Instead of hands, +it has "signal distributors," "central pulse distributors," +"magnetic latching relays," and "reed switches," which complete +and break the calls. Instead of a brain, it has a "central processor." +Instead of an instruction manual, it has a program. Instead of +a handwritten logbook for recording and billing calls, +it has magnetic tapes. And it never has to talk to anybody. +Everything a customer might say to it is done by punching +the direct-dial tone buttons on your subset. + +Although an Electronic Switching Station can't talk, +it does need an interface, some way to relate to its, er, +employers. This interface is known as the "master control +center." (This interface might be better known simply as +"the interface," since it doesn't actually "control" phone +calls directly. However, a term like "Master Control +Center" is just the kind of rhetoric that telco maintenance +engineers--and hackers--find particularly satisfying.) + +Using the master control center, a phone engineer can test +local and trunk lines for malfunctions. He (rarely she) +can check various alarm displays, measure traffic on the lines, +examine the records of telephone usage and the charges for those calls, +and change the programming. + +And, of course, anybody else who gets into the master control center +by remote control can also do these things, if he (rarely she) +has managed to figure them out, or, more likely, has somehow swiped +the knowledge from people who already know. + +In 1989 and 1990, one particular RBOC, BellSouth, +which felt particularly troubled, spent a purported $1.2 +million on computer security. Some think it spent as +much as two million, if you count all the associated costs. +Two million dollars is still very little compared to the +great cost-saving utility of telephonic computer systems. + +Unfortunately, computers are also stupid. +Unlike human beings, computers possess the truly +profound stupidity of the inanimate. + +In the 1960s, in the first shocks of spreading computerization, +there was much easy talk about the stupidity of computers-- +how they could "only follow the program" and were rigidly required +to do "only what they were told." There has been rather less talk +about the stupidity of computers since they began to achieve +grandmaster status in chess tournaments, and to manifest +many other impressive forms of apparent cleverness. + +Nevertheless, computers STILL are profoundly brittle and stupid; +they are simply vastly more subtle in their stupidity and brittleness. +The computers of the 1990s are much more reliable in their components +than earlier computer systems, but they are also called upon to do +far more complex things, under far more challenging conditions. + +On a basic mathematical level, every single line of +a software program offers a chance for some possible screwup. +Software does not sit still when it works; it "runs," +it interacts with itself and with its own inputs and outputs. +By analogy, it stretches like putty into millions of possible +shapes and conditions, so many shapes that they can never +all be successfully tested, not even in the lifespan of the universe. +Sometimes the putty snaps. + +The stuff we call "software" is not like anything that human society +is used to thinking about. Software is something like a machine, +and something like mathematics, and something like language, and +something like thought, and art, and information. . . . But software +is not in fact any of those other things. The protean quality +of software is one of the great sources of its fascination. +It also makes software very powerful, very subtle, +very unpredictable, and very risky. + +Some software is bad and buggy. Some is "robust," +even "bulletproof." The best software is that which has +been tested by thousands of users under thousands of +different conditions, over years. It is then known as +"stable." This does NOT mean that the software is +now flawless, free of bugs. It generally means that there +are plenty of bugs in it, but the bugs are well-identified +and fairly well understood. + +There is simply no way to assure that software is free +of flaws. Though software is mathematical in nature, +it cannot by "proven" like a mathematical theorem; +software is more like language, with inherent ambiguities, +with different definitions, different assumptions, +different levels of meaning that can conflict. + +Human beings can manage, more or less, with +human language because we can catch the gist of it. + +Computers, despite years of effort in "artificial intelligence," +have proven spectacularly bad in "catching the gist" of anything at all. +The tiniest bit of semantic grit may still bring the mightiest computer +tumbling down. One of the most hazardous things you can do to a +computer program is try to improve it--to try to make it safer. +Software "patches" represent new, untried un-"stable" software, +which is by definition riskier. + +The modern telephone system has come to depend, +utterly and irretrievably, upon software. And the +System Crash of January 15, 1990, was caused by an +IMPROVEMENT in software. Or rather, an ATTEMPTED +improvement. + +As it happened, the problem itself--the problem per se--took this form. +A piece of telco software had been written in C language, a standard +language of the telco field. Within the C software was a +long "do. . .while" construct. The "do. . .while" construct +contained a "switch" statement. The "switch" statement contained +an "if" clause. The "if" clause contained a "break." The "break" +was SUPPOSED to "break" the "if clause." Instead, the "break" +broke the "switch" statement. + +That was the problem, the actual reason why people picking up phones +on January 15, 1990, could not talk to one another. + +Or at least, that was the subtle, abstract, cyberspatial +seed of the problem. This is how the problem manifested itself +from the realm of programming into the realm of real life. + +The System 7 software for AT&T's 4ESS switching station, +the "Generic 44E14 Central Office Switch Software," +had been extensively tested, and was considered very stable. +By the end of 1989, eighty of AT&T's switching systems +nationwide had been programmed with the new software. Cautiously, +thirty-four stations were left to run the slower, less-capable +System 6, because AT&T suspected there might be shakedown problems +with the new and unprecedently sophisticated System 7 network. + +The stations with System 7 were programmed to switch over to a backup net +in case of any problems. In mid-December 1989, however, a new high-velocity, +high-security software patch was distributed to each of the 4ESS switches +that would enable them to switch over even more quickly, making the System 7 +network that much more secure. + +Unfortunately, every one of these 4ESS switches was now in possession +of a small but deadly flaw. + +In order to maintain the network, switches must monitor +the condition of other switches--whether they are up and running, +whether they have temporarily shut down, whether they are overloaded +and in need of assistance, and so forth. The new software helped +control this bookkeeping function by monitoring the status calls +from other switches. + +It only takes four to six seconds for a troubled 4ESS switch +to rid itself of all its calls, drop everything temporarily, +and re-boot its software from scratch. Starting over from scratch +will generally rid the switch of any software problems that may have +developed in the course of running the system. Bugs that arise will +be simply wiped out by this process. It is a clever idea. This process +of automatically re-booting from scratch is known as the "normal fault +recovery routine." Since AT&T's software is in fact exceptionally stable, +systems rarely have to go into "fault recovery" in the first place; +but AT&T has always boasted of its "real world" reliability, and this +tactic is a belt-and-suspenders routine. + +The 4ESS switch used its new software to monitor its fellow switches +as they recovered from faults. As other switches came back on line +after recovery, they would send their "OK" signals to the switch. +The switch would make a little note to that effect in its "status map," +recognizing that the fellow switch was back and ready to go, +and should be sent some calls and put back to regular work. + +Unfortunately, while it was busy bookkeeping with the status map, +the tiny flaw in the brand-new software came into play. +The flaw caused the 4ESS switch to interact, subtly but drastically, +with incoming telephone calls from human users. If--and only if-- +two incoming phone-calls happened to hit the switch within a hundredth +of a second, then a small patch of data would be garbled by the flaw. + +But the switch had been programmed to monitor itself +constantly for any possible damage to its data. +When the switch perceived that its data had been somehow garbled, +then it too would go down, for swift repairs to its software. +It would signal its fellow switches not to send any more work. +It would go into the fault-recovery mode for four to six seconds. +And then the switch would be fine again, and would send out its "OK, +ready for work" signal. + +However, the "OK, ready for work" signal was the VERY THING THAT +HAD CAUSED THE SWITCH TO GO DOWN IN THE FIRST PLACE. And ALL the +System 7 switches had the same flaw in their status-map software. +As soon as they stopped to make the bookkeeping note that their fellow +switch was "OK," then they too would become vulnerable to the slight +chance that two phone-calls would hit them within a hundredth of a second. + +At approximately 2:25 P.M. EST on Monday, January 15, +one of AT&T's 4ESS toll switching systems in New York City +had an actual, legitimate, minor problem. It went into fault +recovery routines, announced "I'm going down," then announced, +"I'm back, I'm OK." And this cheery message then blasted +throughout the network to many of its fellow 4ESS switches. + +Many of the switches, at first, completely escaped trouble. +These lucky switches were not hit by the coincidence of +two phone calls within a hundredth of a second. +Their software did not fail--at first. But three switches-- +in Atlanta, St. Louis, and Detroit--were unlucky, +and were caught with their hands full. And they went down. +And they came back up, almost immediately. And they too began +to broadcast the lethal message that they, too, were "OK" again, +activating the lurking software bug in yet other switches. + +As more and more switches did have that bit of bad luck +and collapsed, the call-traffic became more and more densely +packed in the remaining switches, which were groaning +to keep up with the load. And of course, as the calls +became more densely packed, the switches were MUCH MORE LIKELY +to be hit twice within a hundredth of a second. + +It only took four seconds for a switch to get well. +There was no PHYSICAL damage of any kind to the switches, +after all. Physically, they were working perfectly. +This situation was "only" a software problem. + +But the 4ESS switches were leaping up and down every +four to six seconds, in a virulent spreading wave all over America, +in utter, manic, mechanical stupidity. They kept KNOCKING +one another down with their contagious "OK" messages. + +It took about ten minutes for the chain reaction to cripple the network. +Even then, switches would periodically luck-out and manage to resume +their normal work. Many calls--millions of them--were managing +to get through. But millions weren't. + +The switching stations that used System 6 were not directly affected. +Thanks to these old-fashioned switches, AT&T's national system avoided +complete collapse. This fact also made it clear to engineers that +System 7 was at fault. + +Bell Labs engineers, working feverishly in New Jersey, Illinois, +and Ohio, first tried their entire repertoire of standard network +remedies on the malfunctioning System 7. None of the remedies worked, +of course, because nothing like this had ever happened to any +phone system before. + +By cutting out the backup safety network entirely, +they were able to reduce the frenzy of "OK" messages +by about half. The system then began to recover, as the +chain reaction slowed. By 11:30 P.M. on Monday January +15, sweating engineers on the midnight shift breathed a +sigh of relief as the last switch cleared-up. + +By Tuesday they were pulling all the brand-new 4ESS software +and replacing it with an earlier version of System 7. + +If these had been human operators, rather than +computers at work, someone would simply have +eventually stopped screaming. It would have been +OBVIOUS that the situation was not "OK," and common +sense would have kicked in. Humans possess common sense-- +at least to some extent. Computers simply don't. + +On the other hand, computers can handle hundreds +of calls per second. Humans simply can't. If every single +human being in America worked for the phone company, +we couldn't match the performance of digital switches: +direct-dialling, three-way calling, speed-calling, call- +waiting, Caller ID, all the rest of the cornucopia +of digital bounty. Replacing computers with operators +is simply not an option any more. + +And yet we still, anachronistically, expect humans to +be running our phone system. It is hard for us +to understand that we have sacrificed huge amounts +of initiative and control to senseless yet powerful machines. +When the phones fail, we want somebody to be responsible. +We want somebody to blame. + +When the Crash of January 15 happened, the American populace +was simply not prepared to understand that enormous landslides +in cyberspace, like the Crash itself, can happen, +and can be nobody's fault in particular. It was easier to believe, +maybe even in some odd way more reassuring to believe, +that some evil person, or evil group, had done this to us. +"Hackers" had done it. With a virus. A trojan horse. +A software bomb. A dirty plot of some kind. People believed this, +responsible people. In 1990, they were looking hard for evidence +to confirm their heartfelt suspicions. + +And they would look in a lot of places. + +Come 1991, however, the outlines of an apparent new reality +would begin to emerge from the fog. + +On July 1 and 2, 1991, computer-software collapses +in telephone switching stations disrupted service in +Washington DC, Pittsburgh, Los Angeles and San Francisco. +Once again, seemingly minor maintenance problems had +crippled the digital System 7. About twelve million +people were affected in the Crash of July 1, 1991. + +Said the New York Times Service: "Telephone company executives +and federal regulators said they were not ruling out the possibility +of sabotage by computer hackers, but most seemed to think the problems +stemmed from some unknown defect in the software running the networks." + +And sure enough, within the week, a red-faced software company, +DSC Communications Corporation of Plano, Texas, owned up +to "glitches" in the "signal transfer point" software that +DSC had designed for Bell Atlantic and Pacific Bell. +The immediate cause of the July 1 Crash was a single +mistyped character: one tiny typographical flaw +in one single line of the software. One mistyped letter, +in one single line, had deprived the nation's capital of phone service. +It was not particularly surprising that this tiny flaw had escaped attention: +a typical System 7 station requires TEN MILLION lines of code. + +On Tuesday, September 17, 1991, came the most spectacular outage yet. +This case had nothing to do with software failures--at least, not directly. +Instead, a group of AT&T's switching stations in New York City had simply +run out of electrical power and shut down cold. Their back-up batteries +had failed. Automatic warning systems were supposed to warn of the loss +of battery power, but those automatic systems had failed as well. + +This time, Kennedy, La Guardia, and Newark airports +all had their voice and data communications cut. +This horrifying event was particularly ironic, as attacks +on airport computers by hackers had long been a standard +nightmare scenario, much trumpeted by computer-security +experts who feared the computer underground. There had even +been a Hollywood thriller about sinister hackers ruining +airport computers--DIE HARD II. + +Now AT&T itself had crippled airports with computer malfunctions-- +not just one airport, but three at once, some of the busiest in the world. + +Air traffic came to a standstill throughout the Greater New York area, +causing more than 500 flights to be cancelled, in a spreading wave +all over America and even into Europe. Another 500 or so flights +were delayed, affecting, all in all, about 85,000 passengers. +(One of these passengers was the chairman of the Federal +Communications Commission.) + +Stranded passengers in New York and New Jersey were further +infuriated to discover that they could not even manage to +make a long distance phone call, to explain their delay +to loved ones or business associates. Thanks to the crash, +about four and a half million domestic calls, and half a million +international calls, failed to get through. + +The September 17 NYC Crash, unlike the previous ones, +involved not a whisper of "hacker" misdeeds. On the contrary, +by 1991, AT&T itself was suffering much of the vilification +that had formerly been directed at hackers. Congressmen were grumbling. +So were state and federal regulators. And so was the press. + +For their part, ancient rival MCI took out snide full-page +newspaper ads in New York, offering their own long-distance +services for the "next time that AT&T goes down." + +"You wouldn't find a classy company like AT&T using such advertising," +protested AT&T Chairman Robert Allen, unconvincingly. Once again, +out came the full-page AT&T apologies in newspapers, apologies for +"an inexcusable culmination of both human and mechanical failure." +(This time, however, AT&T offered no discount on later calls. +Unkind critics suggested that AT&T were worried about setting any precedent +for refunding the financial losses caused by telephone crashes.) + +Industry journals asked publicly if AT&T was "asleep at the switch." +The telephone network, America's purported marvel of high-tech reliability, +had gone down three times in 18 months. Fortune magazine listed the +Crash of September 17 among the "Biggest Business Goofs of 1991," +cruelly parodying AT&T's ad campaign in an article entitled +"AT&T Wants You Back (Safely On the Ground, God Willing)." + +Why had those New York switching systems simply run out of power? +Because no human being had attended to the alarm system. +Why did the alarm systems blare automatically, +without any human being noticing? Because the three +telco technicians who SHOULD have been listening +were absent from their stations in the power-room, +on another floor of the building--attending a training class. +A training class about the alarm systems for the power room! + +"Crashing the System" was no longer "unprecedented" by late 1991. +On the contrary, it no longer even seemed an oddity. By 1991, +it was clear that all the policemen in the world could no longer +"protect" the phone system from crashes. By far the worst crashes +the system had ever had, had been inflicted, by the system, +upon ITSELF. And this time nobody was making cocksure statements +that this was an anomaly, something that would never happen again. +By 1991 the System's defenders had met their nebulous Enemy, +and the Enemy was--the System. + + + +PART TWO: THE DIGITAL UNDERGROUND + + +The date was May 9, 1990. The Pope was touring Mexico City. +Hustlers from the Medellin Cartel were trying to buy +black-market Stinger missiles in Florida. On the comics page, +Doonesbury character Andy was dying of AIDS. And then. . .a highly +unusual item whose novelty and calculated rhetoric won it +headscratching attention in newspapers all over America. + +The US Attorney's office in Phoenix, Arizona, had issued +a press release announcing a nationwide law enforcement crackdown +against "illegal computer hacking activities." The sweep was +officially known as "Operation Sundevil." + +Eight paragraphs in the press release gave the bare facts: +twenty-seven search warrants carried out on May 8, with three arrests, +and a hundred and fifty agents on the prowl in "twelve" cities across America. +(Different counts in local press reports yielded "thirteen," "fourteen," and +"sixteen" cities.) Officials estimated that criminal losses of revenue +to telephone companies "may run into millions of dollars." Credit for +the Sundevil investigations was taken by the US Secret Service, +Assistant US Attorney Tim Holtzen of Phoenix, and the Assistant +Attorney General of Arizona, Gail Thackeray. + +The prepared remarks of Garry M. Jenkins, appearing in a U.S. Department +of Justice press release, were of particular interest. Mr. Jenkins was the +Assistant Director of the US Secret Service, and the highest-ranking federal +official to take any direct public role in the hacker crackdown of 1990. + +"Today, the Secret Service is sending a clear message to those computer hackers +who have decided to violate the laws of this nation in the mistaken belief +that they can successfully avoid detection by hiding behind the relative +anonymity of their computer terminals. (. . .) "Underground groups have been +formed for the purpose of exchanging information relevant to their criminal +activities. These groups often communicate with each other through message +systems between computers called `bulletin boards.' "Our experience shows +that many computer hacker suspects are no longer misguided teenagers, +mischievously playing games with their computers in their bedrooms. +Some are now high tech computer operators using computers to engage +in unlawful conduct." + +Who were these "underground groups" and "high-tech operators?" +Where had they come from? What did they want? Who WERE they? +Were they "mischievous?" Were they dangerous? How had "misguided teenagers" +managed to alarm the United States Secret Service? And just how widespread +was this sort of thing? + +Of all the major players in the Hacker Crackdown: the phone companies, +law enforcement, the civil libertarians, and the "hackers" themselves-- +the "hackers" are by far the most mysterious, by far the hardest to +understand, by far the WEIRDEST. + +Not only are "hackers" novel in their activities, but they come +in a variety of odd subcultures, with a variety of languages, +motives and values. + +The earliest proto-hackers were probably those unsung mischievous +telegraph boys who were summarily fired by the Bell Company in 1878. + +Legitimate "hackers," those computer enthusiasts who are independent-minded +but law-abiding, generally trace their spiritual ancestry to elite technical +universities, especially M.I.T. and Stanford, in the 1960s. + +But the genuine roots of the modern hacker UNDERGROUND can probably be traced +most successfully to a now much-obscured hippie anarchist movement known as +the Yippies. The Yippies, who took their name from the largely fictional +"Youth International Party," carried out a loud and lively policy of surrealistic +subversion and outrageous political mischief. Their basic tenets were flagrant +sexual promiscuity, open and copious drug use, the political overthrow of any +powermonger over thirty years of age, and an immediate end to the war +in Vietnam, by any means necessary, including the psychic levitation +of the Pentagon. + +The two most visible Yippies were Abbie Hoffman and Jerry Rubin. +Rubin eventually became a Wall Street broker. Hoffman, ardently sought +by federal authorities, went into hiding for seven years, +in Mexico, France, and the United States. While on the lam, +Hoffman continued to write and publish, with help from sympathizers +in the American anarcho-leftist underground. Mostly, Hoffman survived +through false ID and odd jobs. Eventually he underwent facial plastic +surgery and adopted an entirely new identity as one "Barry Freed." +After surrendering himself to authorities in 1980, Hoffman spent a year +in prison on a cocaine conviction. + +Hoffman's worldview grew much darker as the glory days of the 1960s faded. +In 1989, he purportedly committed suicide, under odd and, to some, rather +suspicious circumstances. + +Abbie Hoffman is said to have caused the Federal Bureau of Investigation +to amass the single largest investigation file ever opened on an individual +American citizen. (If this is true, it is still questionable whether the +FBI regarded Abbie Hoffman a serious public threat--quite possibly, +his file was enormous simply because Hoffman left colorful legendry +wherever he went). He was a gifted publicist, who regarded electronic +media as both playground and weapon. He actively enjoyed manipulating +network TV and other gullible, image-hungry media, with various weird lies, +mindboggling rumors, impersonation scams, and other sinister distortions, +all absolutely guaranteed to upset cops, Presidential candidates, +and federal judges. Hoffman's most famous work was a book self-reflexively +known as STEAL THIS BOOK, which publicized a number of methods by which young, +penniless hippie agitators might live off the fat of a system supported by +humorless drones. STEAL THIS BOOK, whose title urged readers to damage +the very means of distribution which had put it into their hands, +might be described as a spiritual ancestor of a computer virus. + +Hoffman, like many a later conspirator, made extensive use of +pay-phones for his agitation work--in his case, generally through +the use of cheap brass washers as coin-slugs. + +During the Vietnam War, there was a federal surtax imposed on telephone +service; Hoffman and his cohorts could, and did, argue that in systematically +stealing phone service they were engaging in civil disobedience: +virtuously denying tax funds to an illegal and immoral war. + +But this thin veil of decency was soon dropped entirely. +Ripping-off the System found its own justification in deep alienation +and a basic outlaw contempt for conventional bourgeois values. +Ingenious, vaguely politicized varieties of rip-off, +which might be described as "anarchy by convenience," +became very popular in Yippie circles, and because rip-off +was so useful, it was to survive the Yippie movement itself. + +In the early 1970s, it required fairly limited expertise +and ingenuity to cheat payphones, to divert "free" +electricity and gas service, or to rob vending machines +and parking meters for handy pocket change. It also required +a conspiracy to spread this knowledge, and the gall +and nerve actually to commit petty theft, but the Yippies +had these qualifications in plenty. In June 1971, Abbie +Hoffman and a telephone enthusiast sarcastically known +as "Al Bell" began publishing a newsletter called Youth +International Party Line. This newsletter was dedicated +to collating and spreading Yippie rip-off techniques, +especially of phones, to the joy of the freewheeling +underground and the insensate rage of all straight people. +As a political tactic, phone-service theft ensured +that Yippie advocates would always have ready access +to the long-distance telephone as a medium, despite +the Yippies' chronic lack of organization, discipline, +money, or even a steady home address. + +PARTY LINE was run out of Greenwich Village for a couple of years, +then "Al Bell" more or less defected from the faltering ranks of Yippiedom, +changing the newsletter's name to TAP or Technical Assistance Program. +After the Vietnam War ended, the steam began leaking rapidly out of American +radical dissent. But by this time, "Bell" and his dozen or so +core contributors had the bit between their teeth, +and had begun to derive tremendous gut-level satisfaction +from the sensation of pure TECHNICAL POWER. + +TAP articles, once highly politicized, became pitilessly jargonized +and technical, in homage or parody to the Bell System's own technical +documents, which TAP studied closely, gutted, and reproduced without +permission. The TAP elite revelled in gloating possession +of the specialized knowledge necessary to beat the system. + +"Al Bell" dropped out of the game by the late 70s, +and "Tom Edison" took over; TAP readers (some 1400 of +them, all told) now began to show more interest in telex +switches and the growing phenomenon of computer systems. + +In 1983, "Tom Edison" had his computer stolen and his house +set on fire by an arsonist. This was an eventually mortal blow +to TAP (though the legendary name was to be resurrected +in 1990 by a young Kentuckian computer-outlaw named "Predat0r.") + +# + +Ever since telephones began to make money, there have been +people willing to rob and defraud phone companies. +The legions of petty phone thieves vastly outnumber those +"phone phreaks" who "explore the system" for the sake +of the intellectual challenge. The New York metropolitan area +(long in the vanguard of American crime) claims over 150,000 +physical attacks on pay telephones every year! Studied carefully, +a modern payphone reveals itself as a little fortress, carefully +designed and redesigned over generations, to resist coin-slugs, +zaps of electricity, chunks of coin-shaped ice, prybars, magnets, +lockpicks, blasting caps. Public pay- phones must survive in a world +of unfriendly, greedy people, and a modern payphone is as exquisitely +evolved as a cactus. +Because the phone network pre-dates the computer network, +the scofflaws known as "phone phreaks" pre-date the scofflaws +known as "computer hackers." In practice, today, the line +between "phreaking" and "hacking" is very blurred, +just as the distinction between telephones and computers +has blurred. The phone system has been digitized, +and computers have learned to "talk" over phone-lines. +What's worse--and this was the point of the Mr. Jenkins +of the Secret Service--some hackers have learned to steal, +and some thieves have learned to hack. + +Despite the blurring, one can still draw a few useful +behavioral distinctions between "phreaks" and "hackers." +Hackers are intensely interested in the "system" per se, +and enjoy relating to machines. "Phreaks" are more +social, manipulating the system in a rough-and-ready +fashion in order to get through to other human beings, +fast, cheap and under the table. + +Phone phreaks love nothing so much as "bridges," +illegal conference calls of ten or twelve chatting +conspirators, seaboard to seaboard, lasting for many hours +--and running, of course, on somebody else's tab, +preferably a large corporation's. + +As phone-phreak conferences wear on, people drop out +(or simply leave the phone off the hook, while they +sashay off to work or school or babysitting), +and new people are phoned up and invited to join in, +from some other continent, if possible. Technical trivia, +boasts, brags, lies, head-trip deceptions, weird rumors, +and cruel gossip are all freely exchanged. + +The lowest rung of phone-phreaking is the theft of telephone access codes. +Charging a phone call to somebody else's stolen number is, of course, +a pig-easy way of stealing phone service, requiring practically no +technical expertise. This practice has been very widespread, +especially among lonely people without much money who are far from home. +Code theft has flourished especially in college dorms, military bases, +and, notoriously, among roadies for rock bands. Of late, code theft +has spread very rapidly among Third Worlders in the US, who pile up +enormous unpaid long-distance bills to the Caribbean, South America, +and Pakistan. + +The simplest way to steal phone-codes is simply to look over +a victim's shoulder as he punches-in his own code-number +on a public payphone. This technique is known as "shoulder-surfing," +and is especially common in airports, bus terminals, and train stations. +The code is then sold by the thief for a few dollars. The buyer abusing +the code has no computer expertise, but calls his Mom in New York, +Kingston or Caracas and runs up a huge bill with impunity. The losses +from this primitive phreaking activity are far, far greater than the +monetary losses caused by computer-intruding hackers. + +In the mid-to-late 1980s, until the introduction of sterner telco +security measures, COMPUTERIZED code theft worked like a charm, +and was virtually omnipresent throughout the digital underground, +among phreaks and hackers alike. This was accomplished through +programming one's computer to try random code numbers over the telephone +until one of them worked. Simple programs to do this were widely available +in the underground; a computer running all night was likely to come up with +a dozen or so useful hits. This could be repeated week after week until +one had a large library of stolen codes. + +Nowadays, the computerized dialling of hundreds of numbers +can be detected within hours and swiftly traced. +If a stolen code is repeatedly abused, this too can +be detected within a few hours. But for years in the 1980s, +the publication of stolen codes was a kind of elementary etiquette +for fledgling hackers. The simplest way to establish your bona-fides +as a raider was to steal a code through repeated random dialling +and offer it to the "community" for use. Codes could be both stolen, +and used, simply and easily from the safety of one's own bedroom, +with very little fear of detection or punishment. + +Before computers and their phone-line modems entered American homes +in gigantic numbers, phone phreaks had their own special telecommunications +hardware gadget, the famous "blue box." This fraud device (now rendered +increasingly useless by the digital evolution of the phone system) could +trick switching systems into granting free access to long-distance lines. +It did this by mimicking the system's own signal, a tone of 2600 hertz. + +Steven Jobs and Steve Wozniak, the founders of Apple Computer, Inc., +once dabbled in selling blue-boxes in college dorms in California. +For many, in the early days of phreaking, blue-boxing was scarcely +perceived as "theft," but rather as a fun (if sneaky) way to use +excess phone capacity harmlessly. After all, the long-distance +lines were JUST SITTING THERE. . . . Whom did it hurt, really? +If you're not DAMAGING the system, and you're not USING UP ANY +TANGIBLE RESOURCE, and if nobody FINDS OUT what you did, +then what real harm have you done? What exactly HAVE you "stolen," +anyway? If a tree falls in the forest and nobody hears it, +how much is the noise worth? Even now this remains a rather +dicey question. + +Blue-boxing was no joke to the phone companies, however. +Indeed, when Ramparts magazine, a radical publication in California, +printed the wiring schematics necessary to create a mute box in June 1972, +the magazine was seized by police and Pacific Bell phone-company officials. +The mute box, a blue-box variant, allowed its user to receive long-distance +calls free of charge to the caller. This device was closely described in a +Ramparts article wryly titled "Regulating the Phone Company In Your Home." +Publication of this article was held to be in violation of Californian +State Penal Code section 502.7, which outlaws ownership of wire-fraud +devices and the selling of "plans or instructions for any instrument, +apparatus, or device intended to avoid telephone toll charges." + +Issues of Ramparts were recalled or seized on the newsstands, +and the resultant loss of income helped put the magazine out of business. +This was an ominous precedent for free-expression issues, but the telco's +crushing of a radical-fringe magazine passed without serious challenge +at the time. Even in the freewheeling California 1970s, it was widely felt +that there was something sacrosanct about what the phone company knew; +that the telco had a legal and moral right to protect itself by shutting +off the flow of such illicit information. Most telco information was so +"specialized" that it would scarcely be understood by any honest member +of the public. If not published, it would not be missed. To print such +material did not seem part of the legitimate role of a free press. + +In 1990 there would be a similar telco-inspired attack +on the electronic phreak/hacking "magazine" Phrack. +The Phrack legal case became a central issue in the +Hacker Crackdown, and gave rise to great controversy. +Phrack would also be shut down, for a time, at least, +but this time both the telcos and their law-enforcement +allies would pay a much larger price for their actions. +The Phrack case will be examined in detail, later. + +Phone-phreaking as a social practice is still very +much alive at this moment. Today, phone-phreaking +is thriving much more vigorously than the better-known +and worse-feared practice of "computer hacking." +New forms of phreaking are spreading rapidly, following +new vulnerabilities in sophisticated phone services. + +Cellular phones are especially vulnerable; their chips +can be re-programmed to present a false caller ID +and avoid billing. Doing so also avoids police tapping, +making cellular-phone abuse a favorite among drug-dealers. +"Call-sell operations" using pirate cellular phones can, +and have, been run right out of the backs of cars, which move +from "cell" to "cell" in the local phone system, retailing +stolen long-distance service, like some kind of demented +electronic version of the neighborhood ice-cream truck. + +Private branch-exchange phone systems in large corporations +can be penetrated; phreaks dial-up a local company, enter its +internal phone-system, hack it, then use the company's own +PBX system to dial back out over the public network, +causing the company to be stuck with the resulting +long-distance bill. This technique is known as "diverting." +"Diverting" can be very costly, especially because phreaks +tend to travel in packs and never stop talking. +Perhaps the worst by-product of this "PBX fraud" +is that victim companies and telcos have sued one another +over the financial responsibility for the stolen calls, +thus enriching not only shabby phreaks but well-paid lawyers. + +"Voice-mail systems" can also be abused; phreaks +can seize their own sections of these sophisticated +electronic answering machines, and use them for trading +codes or knowledge of illegal techniques. Voice-mail +abuse does not hurt the company directly, but finding +supposedly empty slots in your company's answering +machine all crammed with phreaks eagerly chattering +and hey-duding one another in impenetrable jargon can +cause sensations of almost mystical repulsion and dread. + +Worse yet, phreaks have sometimes been known to react +truculently to attempts to "clean up" the voice-mail system. +Rather than humbly acquiescing to being thrown out of their playground, +they may very well call up the company officials at work (or at home) +and loudly demand free voice-mail addresses of their very own. +Such bullying is taken very seriously by spooked victims. + +Acts of phreak revenge against straight people are rare, +but voice-mail systems are especially tempting and vulnerable, +and an infestation of angry phreaks in one's voice-mail system is no joke. +They can erase legitimate messages; or spy on private messages; +or harass users with recorded taunts and obscenities. +They've even been known to seize control of voice-mail security, +and lock out legitimate users, or even shut down the system entirely. + +Cellular phone-calls, cordless phones, and ship-to-shore +telephony can all be monitored by various forms of radio; +this kind of "passive monitoring" is spreading explosively today. +Technically eavesdropping on other people's cordless and cellular +phone-calls is the fastest-growing area in phreaking today. +This practice strongly appeals to the lust for power and conveys +gratifying sensations of technical superiority over the eavesdropping +victim. Monitoring is rife with all manner of tempting evil mischief. +Simple prurient snooping is by far the most common activity. +But credit-card numbers unwarily spoken over the phone can be recorded, +stolen and used. And tapping people's phone-calls (whether through +active telephone taps or passive radio monitors) does lend itself +conveniently to activities like blackmail, industrial espionage, +and political dirty tricks. + +It should be repeated that telecommunications fraud, +the theft of phone service, causes vastly greater monetary +losses than the practice of entering into computers by stealth. +Hackers are mostly young suburban American white males, +and exist in their hundreds--but "phreaks" come from both sexes +and from many nationalities, ages and ethnic backgrounds, +and are flourishing in the thousands. + +# + +The term "hacker" has had an unfortunate history. +This book, The Hacker Crackdown, has little to say about +"hacking" in its finer, original sense. The term can signify +the free-wheeling intellectual exploration of the highest +and deepest potential of computer systems. Hacking can +describe the determination to make access to computers +and information as free and open as possible. Hacking +can involve the heartfelt conviction that beauty can +be found in computers, that the fine aesthetic in a perfect +program can liberate the mind and spirit. This is "hacking" +as it was defined in Steven Levy's much-praised history +of the pioneer computer milieu, Hackers, published in 1984. + +Hackers of all kinds are absolutely soaked through with heroic +anti-bureaucratic sentiment. Hackers long for recognition +as a praiseworthy cultural archetype, the postmodern electronic +equivalent of the cowboy and mountain man. Whether they deserve +such a reputation is something for history to decide. But many hackers-- +including those outlaw hackers who are computer intruders, and whose +activities are defined as criminal--actually attempt to LIVE UP TO +this techno-cowboy reputation. And given that electronics and +telecommunications are still largely unexplored territories, +there is simply NO TELLING what hackers might uncover. + +For some people, this freedom is the very breath of oxygen, +the inventive spontaneity that makes life worth living +and that flings open doors to marvellous possibility and +individual empowerment. But for many people +--and increasingly so--the hacker is an ominous figure, +a smart-aleck sociopath ready to burst out of his basement +wilderness and savage other people's lives for his own +anarchical convenience. + +Any form of power without responsibility, without direct +and formal checks and balances, is frightening to people-- +and reasonably so. It should be frankly admitted that +hackers ARE frightening, and that the basis of this fear +is not irrational. + +Fear of hackers goes well beyond the fear of merely criminal activity. + +Subversion and manipulation of the phone system +is an act with disturbing political overtones. +In America, computers and telephones are potent symbols +of organized authority and the technocratic business elite. + +But there is an element in American culture that +has always strongly rebelled against these symbols; +rebelled against all large industrial computers +and all phone companies. A certain anarchical tinge deep +in the American soul delights in causing confusion and pain +to all bureaucracies, including technological ones. + +There is sometimes malice and vandalism in this attitude, +but it is a deep and cherished part of the American national character. +The outlaw, the rebel, the rugged individual, the pioneer, +the sturdy Jeffersonian yeoman, the private citizen resisting +interference in his pursuit of happiness--these are figures that all +Americans recognize, and that many will strongly applaud and defend. + +Many scrupulously law-abiding citizens today do cutting-edge work +with electronics--work that has already had tremendous social influence +and will have much more in years to come. In all truth, these talented, +hardworking, law-abiding, mature, adult people are far more disturbing +to the peace and order of the current status quo than any scofflaw group +of romantic teenage punk kids. These law-abiding hackers have the power, +ability, and willingness to influence other people's lives quite unpredictably. +They have means, motive, and opportunity to meddle drastically with the +American social order. When corralled into governments, universities, +or large multinational companies, and forced to follow rulebooks +and wear suits and ties, they at least have some conventional halters +on their freedom of action. But when loosed alone, or in small groups, +and fired by imagination and the entrepreneurial spirit, they can move +mountains--causing landslides that will likely crash directly into your +office and living room. + +These people, as a class, instinctively recognize that a public, +politicized attack on hackers will eventually spread to them-- +that the term "hacker," once demonized, might be used to knock +their hands off the levers of power and choke them out of existence. +There are hackers today who fiercely and publicly resist any besmirching +of the noble title of hacker. Naturally and understandably, they deeply +resent the attack on their values implicit in using the word "hacker" +as a synonym for computer-criminal. + +This book, sadly but in my opinion unavoidably, rather adds +to the degradation of the term. It concerns itself mostly with "hacking" +in its commonest latter-day definition, i.e., intruding into computer +systems by stealth and without permission. The term "hacking" is used +routinely today by almost all law enforcement officials with any +professional interest in computer fraud and abuse. American police +describe almost any crime committed with, by, through, or against +a computer as hacking. + +Most importantly, "hacker" is what computer-intruders +choose to call THEMSELVES. Nobody who "hacks" into systems +willingly describes himself (rarely, herself) as a "computer intruder," +"computer trespasser," "cracker," "wormer," "darkside hacker" +or "high tech street gangster." Several other demeaning terms +have been invented in the hope that the press and public +will leave the original sense of the word alone. But few people +actually use these terms. (I exempt the term "cyberpunk," +which a few hackers and law enforcement people actually do use. +The term "cyberpunk" is drawn from literary criticism and has +some odd and unlikely resonances, but, like hacker, +cyberpunk too has become a criminal pejorative today.) + +In any case, breaking into computer systems was hardly alien +to the original hacker tradition. The first tottering systems +of the 1960s required fairly extensive internal surgery merely +to function day-by-day. Their users "invaded" the deepest, +most arcane recesses of their operating software almost +as a matter of routine. "Computer security" in these early, +primitive systems was at best an afterthought. What security +there was, was entirely physical, for it was assumed that +anyone allowed near this expensive, arcane hardware would be +a fully qualified professional expert. + +In a campus environment, though, this meant that grad students, +teaching assistants, undergraduates, and eventually, +all manner of dropouts and hangers-on ended up accessing +and often running the works. + +Universities, even modern universities, are not in +the business of maintaining security over information. +On the contrary, universities, as institutions, pre-date +the "information economy" by many centuries and are not- +for-profit cultural entities, whose reason for existence +(purportedly) is to discover truth, codify it through +techniques of scholarship, and then teach it. Universities +are meant to PASS THE TORCH OF CIVILIZATION, not just +download data into student skulls, and the values of the +academic community are strongly at odds with those of all +would-be information empires. Teachers at all levels, from +kindergarten up, have proven to be shameless and persistent +software and data pirates. Universities do not merely +"leak information" but vigorously broadcast free thought. + +This clash of values has been fraught with controversy. +Many hackers of the 1960s remember their professional +apprenticeship as a long guerilla war against the uptight +mainframe-computer "information priesthood." These computer-hungry +youngsters had to struggle hard for access to computing power, +and many of them were not above certain, er, shortcuts. +But, over the years, this practice freed computing +from the sterile reserve of lab-coated technocrats and +was largely responsible for the explosive growth of computing +in general society--especially PERSONAL computing. + +Access to technical power acted like catnip on certain +of these youngsters. Most of the basic techniques of +computer intrusion: password cracking, trapdoors, backdoors, +trojan horses--were invented in college environments in the 1960s, +in the early days of network computing. Some off-the-cuff +experience at computer intrusion was to be in the informal +resume of most "hackers" and many future industry giants. +Outside of the tiny cult of computer enthusiasts, few people +thought much about the implications of "breaking into" +computers. This sort of activity had not yet been publicized, +much less criminalized. + +In the 1960s, definitions of "property" and "privacy" +had not yet been extended to cyberspace. Computers +were not yet indispensable to society. There were no vast +databanks of vulnerable, proprietary information stored +in computers, which might be accessed, copied without +permission, erased, altered, or sabotaged. The stakes +were low in the early days--but they grew every year, +exponentially, as computers themselves grew. + +By the 1990s, commercial and political pressures +had become overwhelming, and they broke the social +boundaries of the hacking subculture. Hacking +had become too important to be left to the hackers. +Society was now forced to tackle the intangible nature +of cyberspace-as-property, cyberspace as privately-owned +unreal-estate. In the new, severe, responsible, high-stakes +context of the "Information Society" of the 1990s, +"hacking" was called into question. + +What did it mean to break into a computer without +permission and use its computational power, or look +around inside its files without hurting anything? +What were computer-intruding hackers, anyway--how should +society, and the law, best define their actions? +Were they just BROWSERS, harmless intellectual explorers? +Were they VOYEURS, snoops, invaders of privacy? Should +they be sternly treated as potential AGENTS OF ESPIONAGE, +or perhaps as INDUSTRIAL SPIES? Or were they best +defined as TRESPASSERS, a very common teenage +misdemeanor? Was hacking THEFT OF SERVICE? +(After all, intruders were getting someone else's +computer to carry out their orders, without permission +and without paying). Was hacking FRAUD? Maybe it was +best described as IMPERSONATION. The commonest mode +of computer intrusion was (and is) to swipe or snoop +somebody else's password, and then enter the computer +in the guise of another person--who is commonly stuck +with the blame and the bills. + +Perhaps a medical metaphor was better--hackers should +be defined as "sick," as COMPUTER ADDICTS unable +to control their irresponsible, compulsive behavior. + +But these weighty assessments meant little to the +people who were actually being judged. From inside +the underground world of hacking itself, all these +perceptions seem quaint, wrongheaded, stupid, or meaningless. +The most important self-perception of underground hackers-- +from the 1960s, right through to the present day--is that +they are an ELITE. The day-to-day struggle in the underground +is not over sociological definitions--who cares?--but for power, +knowledge, and status among one's peers. + +When you are a hacker, it is your own inner conviction +of your elite status that enables you to break, or let +us say "transcend," the rules. It is not that ALL rules +go by the board. The rules habitually broken by hackers +are UNIMPORTANT rules--the rules of dopey greedhead telco +bureaucrats and pig-ignorant government pests. + +Hackers have their OWN rules, which separate behavior +which is cool and elite, from behavior which is rodentlike, +stupid and losing. These "rules," however, are mostly unwritten +and enforced by peer pressure and tribal feeling. Like all rules +that depend on the unspoken conviction that everybody else +is a good old boy, these rules are ripe for abuse. The mechanisms +of hacker peer- pressure, "teletrials" and ostracism, are rarely used +and rarely work. Back-stabbing slander, threats, and electronic +harassment are also freely employed in down-and-dirty intrahacker feuds, +but this rarely forces a rival out of the scene entirely. The only real +solution for the problem of an utterly losing, treacherous and rodentlike +hacker is to TURN HIM IN TO THE POLICE. Unlike the Mafia or Medellin Cartel, +the hacker elite cannot simply execute the bigmouths, creeps and troublemakers +among their ranks, so they turn one another in with astonishing frequency. + +There is no tradition of silence or OMERTA in the hacker underworld. +Hackers can be shy, even reclusive, but when they do talk, hackers +tend to brag, boast and strut. Almost everything hackers do is INVISIBLE; +if they don't brag, boast, and strut about it, then NOBODY WILL EVER KNOW. +If you don't have something to brag, boast, and strut about, then nobody +in the underground will recognize you and favor you with vital cooperation +and respect. + +The way to win a solid reputation in the underground +is by telling other hackers things that could only +have been learned by exceptional cunning and stealth. +Forbidden knowledge, therefore, is the basic currency +of the digital underground, like seashells among +Trobriand Islanders. Hackers hoard this knowledge, +and dwell upon it obsessively, and refine it, +and bargain with it, and talk and talk about it. + +Many hackers even suffer from a strange obsession to TEACH-- +to spread the ethos and the knowledge of the digital underground. +They'll do this even when it gains them no particular advantage +and presents a grave personal risk. + +And when that risk catches up with them, they will go right on teaching +and preaching--to a new audience this time, their interrogators from law +enforcement. Almost every hacker arrested tells everything he knows-- +all about his friends, his mentors, his disciples--legends, threats, +horror stories, dire rumors, gossip, hallucinations. This is, of course, +convenient for law enforcement--except when law enforcement begins +to believe hacker legendry. + +Phone phreaks are unique among criminals in their willingness +to call up law enforcement officials--in the office, at their homes-- +and give them an extended piece of their mind. It is hard not to +interpret this as BEGGING FOR ARREST, and in fact it is an act +of incredible foolhardiness. Police are naturally nettled +by these acts of chutzpah and will go well out of their way +to bust these flaunting idiots. But it can also be interpreted +as a product of a world-view so elitist, so closed and hermetic, +that electronic police are simply not perceived as "police," +but rather as ENEMY PHONE PHREAKS who should be scolded +into behaving "decently." + +Hackers at their most grandiloquent perceive themselves +as the elite pioneers of a new electronic world. +Attempts to make them obey the democratically +established laws of contemporary American society are +seen as repression and persecution. After all, they argue, +if Alexander Graham Bell had gone along with the rules +of the Western Union telegraph company, there would have +been no telephones. If Jobs and Wozniak had believed +that IBM was the be-all and end-all, there would have +been no personal computers. If Benjamin Franklin and +Thomas Jefferson had tried to "work within the system" +there would have been no United States. + +Not only do hackers privately believe this as an article of faith, +but they have been known to write ardent manifestos about it. +Here are some revealing excerpts from an especially vivid hacker manifesto: +"The Techno-Revolution" by "Dr. Crash," which appeared in electronic +form in Phrack Volume 1, Issue 6, Phile 3. + + +"To fully explain the true motives behind hacking, +we must first take a quick look into the past. In the 1960s, +a group of MIT students built the first modern computer system. +This wild, rebellious group of young men were the first to bear +the name `hackers.' The systems that they developed were intended +to be used to solve world problems and to benefit all of mankind. +"As we can see, this has not been the case. The computer system +has been solely in the hands of big businesses and the government. +The wonderful device meant to enrich life has become a weapon which +dehumanizes people. To the government and large businesses, +people are no more than disk space, and the government doesn't +use computers to arrange aid for the poor, but to control nuclear +death weapons. The average American can only have access +to a small microcomputer which is worth only a fraction +of what they pay for it. The businesses keep the +true state-of-the-art equipment away from the people +behind a steel wall of incredibly high prices and bureaucracy. +It is because of this state of affairs that hacking was born. (. . .) +"Of course, the government doesn't want the monopoly of technology broken, +so they have outlawed hacking and arrest anyone who is caught. (. . .) +The phone company is another example of technology abused and kept +from people with high prices. (. . .) "Hackers often find that their +existing equipment, due to the monopoly tactics of computer companies, +is inefficient for their purposes. Due to the exorbitantly high prices, +it is impossible to legally purchase the necessary equipment. +This need has given still another segment of the fight: Credit Carding. +Carding is a way of obtaining the necessary goods without paying for them. +It is again due to the companies' stupidity that Carding is so easy, +and shows that the world's businesses are in the hands of those +with considerably less technical know-how than we, the hackers. (. . .) +"Hacking must continue. We must train newcomers to the art of hacking. +(. . . .) And whatever you do, continue the fight. Whether you know it +or not, if you are a hacker, you are a revolutionary. Don't worry, +you're on the right side." + +The defense of "carding" is rare. Most hackers regard credit-card +theft as "poison" to the underground, a sleazy and immoral effort that, +worse yet, is hard to get away with. Nevertheless, manifestos advocating +credit-card theft, the deliberate crashing of computer systems, +and even acts of violent physical destruction such as vandalism +and arson do exist in the underground. These boasts and threats +are taken quite seriously by the police. And not every hacker +is an abstract, Platonic computer-nerd. Some few are quite experienced +at picking locks, robbing phone-trucks, and breaking and entering buildings. + +Hackers vary in their degree of hatred for authority +and the violence of their rhetoric. But, at a bottom line, +they are scofflaws. They don't regard the current rules +of electronic behavior as respectable efforts to preserve +law and order and protect public safety. They regard these +laws as immoral efforts by soulless corporations to protect +their profit margins and to crush dissidents. "Stupid" people, +including police, businessmen, politicians, and journalists, +simply have no right to judge the actions of those possessed of genius, +techno-revolutionary intentions, and technical expertise. + +# + +Hackers are generally teenagers and college kids not +engaged in earning a living. They often come from fairly +well-to-do middle-class backgrounds, and are markedly +anti-materialistic (except, that is, when it comes to +computer equipment). Anyone motivated by greed for +mere money (as opposed to the greed for power, +knowledge and status) is swiftly written-off as a narrow- +minded breadhead whose interests can only be corrupt +and contemptible. Having grown up in the 1970s and +1980s, the young Bohemians of the digital underground +regard straight society as awash in plutocratic corruption, +where everyone from the President down is for sale and +whoever has the gold makes the rules. + +Interestingly, there's a funhouse-mirror image of this attitude +on the other side of the conflict. The police are also +one of the most markedly anti-materialistic groups +in American society, motivated not by mere money +but by ideals of service, justice, esprit-de-corps, +and, of course, their own brand of specialized knowledge +and power. Remarkably, the propaganda war between cops +and hackers has always involved angry allegations +that the other side is trying to make a sleazy buck. +Hackers consistently sneer that anti-phreak prosecutors +are angling for cushy jobs as telco lawyers and that +computer-crime police are aiming to cash in later +as well-paid computer-security consultants in the private sector. + +For their part, police publicly conflate all +hacking crimes with robbing payphones with crowbars. +Allegations of "monetary losses" from computer intrusion +are notoriously inflated. The act of illicitly copying +a document from a computer is morally equated with +directly robbing a company of, say, half a million dollars. +The teenage computer intruder in possession of this "proprietary" +document has certainly not sold it for such a sum, would likely +have little idea how to sell it at all, and quite probably +doesn't even understand what he has. He has not made a cent +in profit from his felony but is still morally equated with +a thief who has robbed the church poorbox and lit out for Brazil. + +Police want to believe that all hackers are thieves. +It is a tortuous and almost unbearable act for the American +justice system to put people in jail because they want +to learn things which are forbidden for them to know. +In an American context, almost any pretext for punishment +is better than jailing people to protect certain restricted +kinds of information. Nevertheless, POLICING INFORMATION +is part and parcel of the struggle against hackers. + +This dilemma is well exemplified by the remarkable +activities of "Emmanuel Goldstein," editor and publisher +of a print magazine known as 2600: The Hacker Quarterly. +Goldstein was an English major at Long Island's State University +of New York in the '70s, when he became involved with the local +college radio station. His growing interest in electronics +caused him to drift into Yippie TAP circles and thus into +the digital underground, where he became a self-described +techno-rat. His magazine publishes techniques of computer +intrusion and telephone "exploration" as well as gloating +exposes of telco misdeeds and governmental failings. + +Goldstein lives quietly and very privately in a large, +crumbling Victorian mansion in Setauket, New York. +The seaside house is decorated with telco decals, chunks of +driftwood, and the basic bric-a-brac of a hippie crash-pad. +He is unmarried, mildly unkempt, and survives mostly +on TV dinners and turkey-stuffing eaten straight out +of the bag. Goldstein is a man of considerable charm +and fluency, with a brief, disarming smile and the kind +of pitiless, stubborn, thoroughly recidivist integrity +that America's electronic police find genuinely alarming. + +Goldstein took his nom-de-plume, or "handle," from +a character in Orwell's 1984, which may be taken, +correctly, as a symptom of the gravity of his sociopolitical +worldview. He is not himself a practicing computer +intruder, though he vigorously abets these actions, +especially when they are pursued against large +corporations or governmental agencies. Nor is he a thief, +for he loudly scorns mere theft of phone service, in favor +of "exploring and manipulating the system." He is probably +best described and understood as a DISSIDENT. + +Weirdly, Goldstein is living in modern America +under conditions very similar to those of former +East European intellectual dissidents. In other words, +he flagrantly espouses a value-system that is deeply +and irrevocably opposed to the system of those in power +and the police. The values in 2600 are generally expressed +in terms that are ironic, sarcastic, paradoxical, or just +downright confused. But there's no mistaking their +radically anti-authoritarian tenor. 2600 holds that +technical power and specialized knowledge, of any kind +obtainable, belong by right in the hands of those individuals +brave and bold enough to discover them--by whatever means necessary. +Devices, laws, or systems that forbid access, and the free +spread of knowledge, are provocations that any free +and self-respecting hacker should relentlessly attack. +The "privacy" of governments, corporations and other soulless +technocratic organizations should never be protected +at the expense of the liberty and free initiative +of the individual techno-rat. + +However, in our contemporary workaday world, both governments +and corporations are very anxious indeed to police information +which is secret, proprietary, restricted, confidential, +copyrighted, patented, hazardous, illegal, unethical, +embarrassing, or otherwise sensitive. This makes Goldstein +persona non grata, and his philosophy a threat. + +Very little about the conditions of Goldstein's daily +life would astonish, say, Vaclav Havel. (We may note +in passing that President Havel once had his word-processor +confiscated by the Czechoslovak police.) Goldstein lives +by SAMIZDAT, acting semi-openly as a data-center +for the underground, while challenging the powers-that-be +to abide by their own stated rules: freedom of speech +and the First Amendment. + +Goldstein thoroughly looks and acts the part of techno-rat, +with shoulder-length ringlets and a piratical black +fisherman's-cap set at a rakish angle. He often shows up +like Banquo's ghost at meetings of computer professionals, +where he listens quietly, half-smiling and taking thorough notes. + +Computer professionals generally meet publicly, +and find it very difficult to rid themselves of Goldstein +and his ilk without extralegal and unconstitutional actions. +Sympathizers, many of them quite respectable people +with responsible jobs, admire Goldstein's attitude and +surreptitiously pass him information. An unknown but +presumably large proportion of Goldstein's 2,000-plus +readership are telco security personnel and police, +who are forced to subscribe to 2600 to stay abreast +of new developments in hacking. They thus find themselves +PAYING THIS GUY'S RENT while grinding their teeth in anguish, +a situation that would have delighted Abbie Hoffman +(one of Goldstein's few idols). + +Goldstein is probably the best-known public representative +of the hacker underground today, and certainly the best-hated. +Police regard him as a Fagin, a corrupter of youth, and speak +of him with untempered loathing. He is quite an accomplished gadfly. +After the Martin Luther King Day Crash of 1990, Goldstein, +for instance, adeptly rubbed salt into the wound in the pages of 2600. +"Yeah, it was fun for the phone phreaks as we watched the network crumble," +he admitted cheerfully. "But it was also an ominous sign of what's +to come. . . . Some AT&T people, aided by well-meaning but ignorant media, +were spreading the notion that many companies had the same software +and therefore could face the same problem someday. Wrong. This was +entirely an AT&T software deficiency. Of course, other companies could +face entirely DIFFERENT software problems. But then, so too could AT&T." + +After a technical discussion of the system's failings, +the Long Island techno-rat went on to offer thoughtful +criticism to the gigantic multinational's hundreds of +professionally qualified engineers. "What we don't know +is how a major force in communications like AT&T could +be so sloppy. What happened to backups? Sure, +computer systems go down all the time, but people +making phone calls are not the same as people logging +on to computers. We must make that distinction. It's not +acceptable for the phone system or any other essential +service to `go down.' If we continue to trust technology +without understanding it, we can look forward to many +variations on this theme. + +"AT&T owes it to its customers to be prepared to INSTANTLY +switch to another network if something strange and unpredictable +starts occurring. The news here isn't so much the failure +of a computer program, but the failure of AT&T's entire structure." + +The very idea of this. . . . this PERSON. . . . offering +"advice" about "AT&T's entire structure" is more than +some people can easily bear. How dare this near-criminal +dictate what is or isn't "acceptable" behavior from AT&T? +Especially when he's publishing, in the very same issue, +detailed schematic diagrams for creating various switching-network +signalling tones unavailable to the public. + +"See what happens when you drop a `silver box' tone or two +down your local exchange or through different long distance +service carriers," advises 2600 contributor "Mr. Upsetter" +in "How To Build a Signal Box." "If you experiment systematically +and keep good records, you will surely discover something interesting." + +This is, of course, the scientific method, generally regarded +as a praiseworthy activity and one of the flowers of modern civilization. +One can indeed learn a great deal with this sort of structured +intellectual activity. Telco employees regard this mode of "exploration" +as akin to flinging sticks of dynamite into their pond to see what lives +on the bottom. + +2600 has been published consistently since 1984. +It has also run a bulletin board computer system, +printed 2600 T-shirts, taken fax calls. . . . +The Spring 1991 issue has an interesting announcement on page 45: +"We just discovered an extra set of wires attached to our fax line +and heading up the pole. (They've since been clipped.) +Your faxes to us and to anyone else could be monitored." +In the worldview of 2600, the tiny band of techno-rat brothers +(rarely, sisters) are a beseiged vanguard of the truly free and honest. +The rest of the world is a maelstrom of corporate crime and high-level +governmental corruption, occasionally tempered with well-meaning +ignorance. To read a few issues in a row is to enter a nightmare +akin to Solzhenitsyn's, somewhat tempered by the fact that 2600 +is often extremely funny. + +Goldstein did not become a target of the Hacker Crackdown, +though he protested loudly, eloquently, and publicly about it, +and it added considerably to his fame. It was not that he is not +regarded as dangerous, because he is so regarded. Goldstein has had +brushes with the law in the past: in 1985, a 2600 bulletin board +computer was seized by the FBI, and some software on it was formally +declared "a burglary tool in the form of a computer program." +But Goldstein escaped direct repression in 1990, because his +magazine is printed on paper, and recognized as subject +to Constitutional freedom of the press protection. +As was seen in the Ramparts case, this is far from +an absolute guarantee. Still, as a practical matter, +shutting down 2600 by court-order would create so much +legal hassle that it is simply unfeasible, at least +for the present. Throughout 1990, both Goldstein +and his magazine were peevishly thriving. + +Instead, the Crackdown of 1990 would concern itself +with the computerized version of forbidden data. +The crackdown itself, first and foremost, was about +BULLETIN BOARD SYSTEMS. Bulletin Board Systems, most often +known by the ugly and un-pluralizable acronym "BBS," are +the life-blood of the digital underground. Boards were +also central to law enforcement's tactics and strategy +in the Hacker Crackdown. + +A "bulletin board system" can be formally defined as +a computer which serves as an information and message- +passing center for users dialing-up over the phone-lines +through the use of modems. A "modem," or modulator- +demodulator, is a device which translates the digital +impulses of computers into audible analog telephone +signals, and vice versa. Modems connect computers +to phones and thus to each other. + +Large-scale mainframe computers have been connected since the 1960s, +but PERSONAL computers, run by individuals out of their homes, +were first networked in the late 1970s. The "board" created +by Ward Christensen and Randy Suess in February 1978, +in Chicago, Illinois, is generally regarded as the first +personal-computer bulletin board system worthy of the name. + +Boards run on many different machines, employing many +different kinds of software. Early boards were crude and buggy, +and their managers, known as "system operators" or "sysops," +were hard-working technical experts who wrote their own software. +But like most everything else in the world of electronics, +boards became faster, cheaper, better-designed, and generally +far more sophisticated throughout the 1980s. They also moved +swiftly out of the hands of pioneers and into those of the +general public. By 1985 there were something in the +neighborhood of 4,000 boards in America. By 1990 it was +calculated, vaguely, that there were about 30,000 boards in +the US, with uncounted thousands overseas. + +Computer bulletin boards are unregulated enterprises. +Running a board is a rough-and-ready, catch-as-catch-can proposition. +Basically, anybody with a computer, modem, software and a phone-line +can start a board. With second-hand equipment and public-domain +free software, the price of a board might be quite small-- +less than it would take to publish a magazine or even a +decent pamphlet. Entrepreneurs eagerly sell bulletin-board +software, and will coach nontechnical amateur sysops in its use. + +Boards are not "presses." They are not magazines, +or libraries, or phones, or CB radios, or traditional cork +bulletin boards down at the local laundry, though they +have some passing resemblance to those earlier media. +Boards are a new medium--they may even be a LARGE NUMBER of new media. + +Consider these unique characteristics: boards are cheap, +yet they can have a national, even global reach. +Boards can be contacted from anywhere in the global +telephone network, at NO COST to the person running the board-- +the caller pays the phone bill, and if the caller is local, +the call is free. Boards do not involve an editorial elite +addressing a mass audience. The "sysop" of a board is not +an exclusive publisher or writer--he is managing an electronic salon, +where individuals can address the general public, play the part +of the general public, and also exchange private mail +with other individuals. And the "conversation" on boards, +though fluid, rapid, and highly interactive, is not spoken, +but written. It is also relatively anonymous, sometimes completely so. + +And because boards are cheap and ubiquitous, regulations +and licensing requirements would likely be practically unenforceable. +It would almost be easier to "regulate," "inspect," and "license" +the content of private mail--probably more so, since the mail system +is operated by the federal government. Boards are run by individuals, +independently, entirely at their own whim. + +For the sysop, the cost of operation is not the primary +limiting factor. Once the investment in a computer and +modem has been made, the only steady cost is the charge +for maintaining a phone line (or several phone lines). +The primary limits for sysops are time and energy. +Boards require upkeep. New users are generally "validated"-- +they must be issued individual passwords, and called at +home by voice-phone, so that their identity can be +verified. Obnoxious users, who exist in plenty, must be +chided or purged. Proliferating messages must be deleted +when they grow old, so that the capacity of the system +is not overwhelmed. And software programs (if such things +are kept on the board) must be examined for possible +computer viruses. If there is a financial charge to use +the board (increasingly common, especially in larger and +fancier systems) then accounts must be kept, and users +must be billed. And if the board crashes--a very common +occurrence--then repairs must be made. + +Boards can be distinguished by the amount of effort +spent in regulating them. First, we have the completely +open board, whose sysop is off chugging brews and +watching re-runs while his users generally degenerate +over time into peevish anarchy and eventual silence. +Second comes the supervised board, where the sysop +breaks in every once in a while to tidy up, calm brawls, +issue announcements, and rid the community of dolts +and troublemakers. Third is the heavily supervised +board, which sternly urges adult and responsible behavior +and swiftly edits any message considered offensive, +impertinent, illegal or irrelevant. And last comes +the completely edited "electronic publication," which +is presented to a silent audience which is not allowed +to respond directly in any way. + +Boards can also be grouped by their degree of anonymity. +There is the completely anonymous board, where everyone +uses pseudonyms--"handles"--and even the sysop is unaware +of the user's true identity. The sysop himself is likely +pseudonymous on a board of this type. Second, and rather +more common, is the board where the sysop knows (or thinks +he knows) the true names and addresses of all users, +but the users don't know one another's names and may not know his. +Third is the board where everyone has to use real names, +and roleplaying and pseudonymous posturing are forbidden. + +Boards can be grouped by their immediacy. "Chat-lines" +are boards linking several users together over several +different phone-lines simultaneously, so that people +exchange messages at the very moment that they type. +(Many large boards feature "chat" capabilities along +with other services.) Less immediate boards, +perhaps with a single phoneline, store messages serially, +one at a time. And some boards are only open for business +in daylight hours or on weekends, which greatly slows response. +A NETWORK of boards, such as "FidoNet," can carry electronic mail +from board to board, continent to continent, across huge distances-- +but at a relative snail's pace, so that a message can take several +days to reach its target audience and elicit a reply. + +Boards can be grouped by their degree of community. +Some boards emphasize the exchange of private, +person-to-person electronic mail. Others emphasize +public postings and may even purge people who "lurk," +merely reading posts but refusing to openly participate. +Some boards are intimate and neighborly. Others are frosty +and highly technical. Some are little more than storage +dumps for software, where users "download" and "upload" programs, +but interact among themselves little if at all. + +Boards can be grouped by their ease of access. Some boards +are entirely public. Others are private and restricted only +to personal friends of the sysop. Some boards divide users by status. +On these boards, some users, especially beginners, strangers or children, +will be restricted to general topics, and perhaps forbidden to post. +Favored users, though, are granted the ability to post as they please, +and to stay "on-line" as long as they like, even to the disadvantage +of other people trying to call in. High-status users can be given access +to hidden areas in the board, such as off-color topics, private discussions, +and/or valuable software. Favored users may even become "remote sysops" +with the power to take remote control of the board through their own +home computers. Quite often "remote sysops" end up doing all the work +and taking formal control of the enterprise, despite the fact that it's +physically located in someone else's house. Sometimes several "co-sysops" +share power. + +And boards can also be grouped by size. Massive, nationwide +commercial networks, such as CompuServe, Delphi, GEnie and Prodigy, +are run on mainframe computers and are generally not considered "boards," +though they share many of their characteristics, such as electronic mail, +discussion topics, libraries of software, and persistent and growing problems +with civil-liberties issues. Some private boards have as many as +thirty phone-lines and quite sophisticated hardware. And then +there are tiny boards. + +Boards vary in popularity. Some boards are huge and crowded, +where users must claw their way in against a constant busy-signal. +Others are huge and empty--there are few things sadder than a formerly +flourishing board where no one posts any longer, and the dead conversations +of vanished users lie about gathering digital dust. Some boards are tiny +and intimate, their telephone numbers intentionally kept confidential +so that only a small number can log on. + +And some boards are UNDERGROUND. + +Boards can be mysterious entities. The activities of +their users can be hard to differentiate from conspiracy. +Sometimes they ARE conspiracies. Boards have harbored, +or have been accused of harboring, all manner of fringe groups, +and have abetted, or been accused of abetting, every manner +of frowned-upon, sleazy, radical, and criminal activity. +There are Satanist boards. Nazi boards. Pornographic boards. +Pedophile boards. Drug- dealing boards. Anarchist boards. +Communist boards. Gay and Lesbian boards (these exist in great profusion, +many of them quite lively with well-established histories). +Religious cult boards. Evangelical boards. Witchcraft +boards, hippie boards, punk boards, skateboarder boards. +Boards for UFO believers. There may well be boards for +serial killers, airline terrorists and professional assassins. +There is simply no way to tell. Boards spring up, flourish, +and disappear in large numbers, in most every corner of +the developed world. Even apparently innocuous public +boards can, and sometimes do, harbor secret areas known +only to a few. And even on the vast, public, commercial services, +private mail is very private--and quite possibly criminal. + +Boards cover most every topic imaginable and some +that are hard to imagine. They cover a vast spectrum +of social activity. However, all board users do have +something in common: their possession of computers +and phones. Naturally, computers and phones are +primary topics of conversation on almost every board. + +And hackers and phone phreaks, those utter devotees +of computers and phones, live by boards. They swarm by boards. +They are bred by boards. By the late 1980s, phone-phreak groups +and hacker groups, united by boards, had proliferated fantastically. + + +As evidence, here is a list of hacker groups compiled +by the editors of Phrack on August 8, 1988. + + +The Administration. +Advanced Telecommunications, Inc. +ALIAS. +American Tone Travelers. +Anarchy Inc. +Apple Mafia. +The Association. +Atlantic Pirates Guild. + +Bad Ass Mother Fuckers. +Bellcore. +Bell Shock Force. +Black Bag. + +Camorra. +C&M Productions. +Catholics Anonymous. +Chaos Computer Club. +Chief Executive Officers. +Circle Of Death. +Circle Of Deneb. +Club X. +Coalition of Hi-Tech +Pirates. +Coast-To-Coast. +Corrupt Computing. +Cult Of The +Dead Cow. +Custom Retaliations. + +Damage Inc. +D&B Communications. +The Danger Gang. +Dec Hunters. +Digital Gang. +DPAK. + +Eastern Alliance. +The Elite Hackers Guild. +Elite Phreakers and Hackers Club. +The Elite Society Of America. +EPG. +Executives Of Crime. +Extasyy Elite. + +Fargo 4A. +Farmers Of Doom. +The Federation. +Feds R Us. +First Class. +Five O. +Five Star. +Force Hackers. +The 414s. + +Hack-A-Trip. +Hackers Of America. +High Mountain Hackers. +High Society. +The Hitchhikers. + +IBM Syndicate. +The Ice Pirates. +Imperial Warlords. +Inner Circle. +Inner Circle II. +Insanity Inc. +International Computer Underground Bandits. + +Justice League of America. + +Kaos Inc. +Knights Of Shadow. +Knights Of The Round Table. + +League Of Adepts. +Legion Of Doom. +Legion Of Hackers. +Lords Of Chaos. +Lunatic Labs, Unlimited. + +Master Hackers. +MAD! +The Marauders. +MD/PhD. + +Metal Communications, Inc. +MetalliBashers, Inc. +MBI. + +Metro Communications. +Midwest Pirates Guild. + +NASA Elite. +The NATO Association. +Neon Knights. + +Nihilist Order. +Order Of The Rose. +OSS. + +Pacific Pirates Guild. +Phantom Access Associates. + +PHido PHreaks. +The Phirm. +Phlash. +PhoneLine Phantoms. +Phone Phreakers Of America. +Phortune 500. + +Phreak Hack Delinquents. +Phreak Hack Destroyers. + +Phreakers, Hackers, And Laundromat Employees Gang (PHALSE Gang). +Phreaks Against Geeks. +Phreaks Against Phreaks Against Geeks. +Phreaks and Hackers of America. +Phreaks Anonymous World Wide. +Project Genesis. +The Punk Mafia. + +The Racketeers. +Red Dawn Text Files. +Roscoe Gang. + + +SABRE. +Secret Circle of Pirates. +Secret Service. +707 Club. +Shadow Brotherhood. +Sharp Inc. +65C02 Elite. + +Spectral Force. +Star League. +Stowaways. +Strata-Crackers. + + +Team Hackers '86. +Team Hackers '87. + +TeleComputist Newsletter Staff. +Tribunal Of Knowledge. + +Triple Entente. +Turn Over And Die Syndrome (TOADS). + +300 Club. +1200 Club. +2300 Club. +2600 Club. +2601 Club. + +2AF. + +The United Soft WareZ Force. +United Technical Underground. + +Ware Brigade. +The Warelords. +WASP. + +Contemplating this list is an impressive, almost humbling business. +As a cultural artifact, the thing approaches poetry. + +Underground groups--subcultures--can be distinguished +from independent cultures by their habit of referring +constantly to the parent society. Undergrounds by their +nature constantly must maintain a membrane of differentiation. +Funny/distinctive clothes and hair, specialized jargon, specialized +ghettoized areas in cities, different hours of rising, working, +sleeping. . . . The digital underground, which specializes in information, +relies very heavily on language to distinguish itself. As can be seen +from this list, they make heavy use of parody and mockery. +It's revealing to see who they choose to mock. + +First, large corporations. We have the Phortune 500, +The Chief Executive Officers, Bellcore, IBM Syndicate, +SABRE (a computerized reservation service maintained +by airlines). The common use of "Inc." is telling-- +none of these groups are actual corporations, +but take clear delight in mimicking them. + +Second, governments and police. NASA Elite, NATO Association. +"Feds R Us" and "Secret Service" are fine bits of fleering boldness. +OSS--the Office of Strategic Services was the forerunner of the CIA. + +Third, criminals. Using stigmatizing pejoratives as a perverse +badge of honor is a time-honored tactic for subcultures: +punks, gangs, delinquents, mafias, pirates, bandits, racketeers. + +Specialized orthography, especially the use of "ph" for "f" +and "z" for the plural "s," are instant recognition symbols. +So is the use of the numeral "0" for the letter "O" +--computer-software orthography generally features a +slash through the zero, making the distinction obvious. + +Some terms are poetically descriptive of computer intrusion: +the Stowaways, the Hitchhikers, the PhoneLine Phantoms, Coast-to-Coast. +Others are simple bravado and vainglorious puffery. +(Note the insistent use of the terms "elite" and "master.") +Some terms are blasphemous, some obscene, others merely cryptic-- +anything to puzzle, offend, confuse, and keep the straights at bay. + +Many hacker groups further re-encrypt their names +by the use of acronyms: United Technical Underground +becomes UTU, Farmers of Doom become FoD, the United SoftWareZ +Force becomes, at its own insistence, "TuSwF," and woe to the +ignorant rodent who capitalizes the wrong letters. + +It should be further recognized that the members of these groups +are themselves pseudonymous. If you did, in fact, run across +the "PhoneLine Phantoms," you would find them to consist of +"Carrier Culprit," "The Executioner," "Black Majik," +"Egyptian Lover," "Solid State," and "Mr Icom." +"Carrier Culprit" will likely be referred to by his friends +as "CC," as in, "I got these dialups from CC of PLP." + +It's quite possible that this entire list refers to as +few as a thousand people. It is not a complete list +of underground groups--there has never been such a list, +and there never will be. Groups rise, flourish, decline, +share membership, maintain a cloud of wannabes and +casual hangers-on. People pass in and out, are ostracized, +get bored, are busted by police, or are cornered by telco +security and presented with huge bills. Many "underground +groups" are software pirates, "warez d00dz," who might break +copy protection and pirate programs, but likely wouldn't dare +to intrude on a computer-system. + +It is hard to estimate the true population of the digital +underground. There is constant turnover. Most hackers +start young, come and go, then drop out at age 22-- +the age of college graduation. And a large majority +of "hackers" access pirate boards, adopt a handle, +swipe software and perhaps abuse a phone-code or two, +while never actually joining the elite. + +Some professional informants, who make it their business +to retail knowledge of the underground to paymasters in private +corporate security, have estimated the hacker population +at as high as fifty thousand. This is likely highly inflated, +unless one counts every single teenage software pirate +and petty phone-booth thief. My best guess is about 5,000 people. +Of these, I would guess that as few as a hundred are truly "elite" +--active computer intruders, skilled enough to penetrate +sophisticated systems and truly to worry corporate security +and law enforcement. + +Another interesting speculation is whether this group +is growing or not. Young teenage hackers are often +convinced that hackers exist in vast swarms and will soon +dominate the cybernetic universe. Older and wiser +veterans, perhaps as wizened as 24 or 25 years old, +are convinced that the glory days are long gone, that the cops +have the underground's number now, and that kids these days +are dirt-stupid and just want to play Nintendo. + +My own assessment is that computer intrusion, as a non-profit act +of intellectual exploration and mastery, is in slow decline, +at least in the United States; but that electronic fraud, +especially telecommunication crime, is growing by leaps and bounds. + +One might find a useful parallel to the digital underground +in the drug underground. There was a time, now much-obscured +by historical revisionism, when Bohemians freely shared joints +at concerts, and hip, small-scale marijuana dealers might +turn people on just for the sake of enjoying a long stoned conversation +about the Doors and Allen Ginsberg. Now drugs are increasingly verboten, +except in a high-stakes, highly-criminal world of highly addictive drugs. +Over years of disenchantment and police harassment, a vaguely ideological, +free-wheeling drug underground has relinquished the business of drug-dealing +to a far more savage criminal hard-core. This is not a pleasant prospect +to contemplate, but the analogy is fairly compelling. + +What does an underground board look like? What distinguishes +it from a standard board? It isn't necessarily the conversation-- +hackers often talk about common board topics, such as hardware, software, +sex, science fiction, current events, politics, movies, personal gossip. +Underground boards can best be distinguished by their files, or "philes," +pre-composed texts which teach the techniques and ethos of the underground. +These are prized reservoirs of forbidden knowledge. Some are anonymous, +but most proudly bear the handle of the "hacker" who has created them, +and his group affiliation, if he has one. + +Here is a partial table-of-contents of philes from an underground board, +somewhere in the heart of middle America, circa 1991. The descriptions +are mostly self-explanatory. + + +BANKAMER.ZIP 5406 06-11-91 Hacking Bank America +CHHACK.ZIP 4481 06-11-91 Chilton Hacking +CITIBANK.ZIP 4118 06-11-91 Hacking Citibank +CREDIMTC.ZIP 3241 06-11-91 Hacking Mtc Credit Company +DIGEST.ZIP 5159 06-11-91 Hackers Digest +HACK.ZIP 14031 06-11-91 How To Hack +HACKBAS.ZIP 5073 06-11-91 Basics Of Hacking +HACKDICT.ZIP 42774 06-11-91 Hackers Dictionary +HACKER.ZIP 57938 06-11-91 Hacker Info +HACKERME.ZIP 3148 06-11-91 Hackers Manual +HACKHAND.ZIP 4814 06-11-91 Hackers Handbook +HACKTHES.ZIP 48290 06-11-91 Hackers Thesis +HACKVMS.ZIP 4696 06-11-91 Hacking Vms Systems +MCDON.ZIP 3830 06-11-91 Hacking Macdonalds (Home Of The Archs) +P500UNIX.ZIP 15525 06-11-91 Phortune 500 Guide To Unix +RADHACK.ZIP 8411 06-11-91 Radio Hacking +TAOTRASH.DOC 4096 12-25-89 Suggestions For Trashing +TECHHACK.ZIP 5063 06-11-91 Technical Hacking + + +The files above are do-it-yourself manuals about computer intrusion. +The above is only a small section of a much larger library of hacking +and phreaking techniques and history. We now move into a different +and perhaps surprising area. + ++------------+ + |Anarchy| ++------------+ + +ANARC.ZIP 3641 06-11-91 Anarchy Files +ANARCHST.ZIP 63703 06-11-91 Anarchist Book +ANARCHY.ZIP 2076 06-11-91 Anarchy At Home +ANARCHY3.ZIP 6982 06-11-91 Anarchy No 3 +ANARCTOY.ZIP 2361 06-11-91 Anarchy Toys +ANTIMODM.ZIP 2877 06-11-91 Anti-modem Weapons +ATOM.ZIP 4494 06-11-91 How To Make An Atom Bomb +BARBITUA.ZIP 3982 06-11-91 Barbiturate Formula +BLCKPWDR.ZIP 2810 06-11-91 Black Powder Formulas +BOMB.ZIP 3765 06-11-91 How To Make Bombs +BOOM.ZIP 2036 06-11-91 Things That Go Boom +CHLORINE.ZIP 1926 06-11-91 Chlorine Bomb +COOKBOOK.ZIP 1500 06-11-91 Anarchy Cook Book +DESTROY.ZIP 3947 06-11-91 Destroy Stuff +DUSTBOMB.ZIP 2576 06-11-91 Dust Bomb +ELECTERR.ZIP 3230 06-11-91 Electronic Terror +EXPLOS1.ZIP 2598 06-11-91 Explosives 1 +EXPLOSIV.ZIP 18051 06-11-91 More Explosives +EZSTEAL.ZIP 4521 06-11-91 Ez-stealing +FLAME.ZIP 2240 06-11-91 Flame Thrower +FLASHLT.ZIP 2533 06-11-91 Flashlight Bomb +FMBUG.ZIP 2906 06-11-91 How To Make An Fm Bug +OMEEXPL.ZIP 2139 06-11-91 Home Explosives +HOW2BRK.ZIP 3332 06-11-91 How To Break In +LETTER.ZIP 2990 06-11-91 Letter Bomb +LOCK.ZIP 2199 06-11-91 How To Pick Locks +MRSHIN.ZIP 3991 06-11-91 Briefcase Locks +NAPALM.ZIP 3563 06-11-91 Napalm At Home +NITRO.ZIP 3158 06-11-91 Fun With Nitro +PARAMIL.ZIP 2962 06-11-91 Paramilitary Info +PICKING.ZIP 3398 06-11-91 Picking Locks +PIPEBOMB.ZIP 2137 06-11-91 Pipe Bomb +POTASS.ZIP 3987 06-11-91 Formulas With Potassium +PRANK.TXT 11074 08-03-90 More Pranks To Pull On Idiots! +REVENGE.ZIP 4447 06-11-91 Revenge Tactics +ROCKET.ZIP 2590 06-11-91 Rockets For Fun +SMUGGLE.ZIP 3385 06-11-91 How To Smuggle + +HOLY COW! The damned thing is full of stuff about bombs! + +What are we to make of this? + +First, it should be acknowledged that spreading +knowledge about demolitions to teenagers is a highly and +deliberately antisocial act. It is not, however, illegal. + +Second, it should be recognized that most of these +philes were in fact WRITTEN by teenagers. Most adult +American males who can remember their teenage years +will recognize that the notion of building a flamethrower +in your garage is an incredibly neat-o idea. ACTUALLY, +building a flamethrower in your garage, however, is +fraught with discouraging difficulty. Stuffing gunpowder +into a booby-trapped flashlight, so as to blow the arm off +your high-school vice-principal, can be a thing of dark +beauty to contemplate. Actually committing assault by +explosives will earn you the sustained attention of the +federal Bureau of Alcohol, Tobacco and Firearms. + +Some people, however, will actually try these plans. +A determinedly murderous American teenager can probably +buy or steal a handgun far more easily than he can brew +fake "napalm" in the kitchen sink. Nevertheless, +if temptation is spread before people, a certain number +will succumb, and a small minority will actually attempt +these stunts. A large minority of that small minority +will either fail or, quite likely, maim themselves, +since these "philes" have not been checked for accuracy, +are not the product of professional experience, +and are often highly fanciful. But the gloating menace +of these philes is not to be entirely dismissed. + +Hackers may not be "serious" about bombing; if they were, +we would hear far more about exploding flashlights, homemade bazookas, +and gym teachers poisoned by chlorine and potassium. +However, hackers are VERY serious about forbidden knowledge. +They are possessed not merely by curiosity, but by +a positive LUST TO KNOW. The desire to know what +others don't is scarcely new. But the INTENSITY +of this desire, as manifested by these young technophilic +denizens of the Information Age, may in fact BE new, +and may represent some basic shift in social values-- +a harbinger of what the world may come to, as society +lays more and more value on the possession, +assimilation and retailing of INFORMATION +as a basic commodity of daily life. + +There have always been young men with obsessive interests +in these topics. Never before, however, have they been able +to network so extensively and easily, and to propagandize +their interests with impunity to random passers-by. +High-school teachers will recognize that there's always +one in a crowd, but when the one in a crowd escapes control +by jumping into the phone-lines, and becomes a hundred such kids +all together on a board, then trouble is brewing visibly. +The urge of authority to DO SOMETHING, even something drastic, +is hard to resist. And in 1990, authority did something. +In fact authority did a great deal. + +# + +The process by which boards create hackers goes something +like this. A youngster becomes interested in computers-- +usually, computer games. He hears from friends that +"bulletin boards" exist where games can be obtained for free. +(Many computer games are "freeware," not copyrighted-- +invented simply for the love of it and given away to the public; +some of these games are quite good.) He bugs his parents for a modem, +or quite often, uses his parents' modem. + +The world of boards suddenly opens up. Computer games +can be quite expensive, real budget-breakers for a kid, +but pirated games, stripped of copy protection, are cheap or free. +They are also illegal, but it is very rare, almost unheard of, +for a small-scale software pirate to be prosecuted. +Once "cracked" of its copy protection, the program, +being digital data, becomes infinitely reproducible. +Even the instructions to the game, any manuals that accompany it, +can be reproduced as text files, or photocopied from legitimate sets. +Other users on boards can give many useful hints in game-playing tactics. +And a youngster with an infinite supply of free computer games can +certainly cut quite a swath among his modem-less friends. + +And boards are pseudonymous. No one need know that you're +fourteen years old--with a little practice at subterfuge, +you can talk to adults about adult things, and be accepted +and taken seriously! You can even pretend to be a girl, +or an old man, or anybody you can imagine. If you find this +kind of deception gratifying, there is ample opportunity +to hone your ability on boards. + +But local boards can grow stale. And almost every board maintains +a list of phone-numbers to other boards, some in distant, tempting, +exotic locales. Who knows what they're up to, in Oregon or Alaska +or Florida or California? It's very easy to find out--just order +the modem to call through its software--nothing to this, just typing +on a keyboard, the same thing you would do for most any computer game. +The machine reacts swiftly and in a few seconds you are talking to +a bunch of interesting people on another seaboard. + +And yet the BILLS for this trivial action can be staggering! +Just by going tippety-tap with your fingers, you may have +saddled your parents with four hundred bucks in long-distance charges, +and gotten chewed out but good. That hardly seems fair. + +How horrifying to have made friends in another state +and to be deprived of their company--and their software-- +just because telephone companies demand absurd amounts of money! +How painful, to be restricted to boards in one's own AREA CODE-- +what the heck is an "area code" anyway, and what makes it so special? +A few grumbles, complaints, and innocent questions of this sort +will often elicit a sympathetic reply from another board user-- +someone with some stolen codes to hand. You dither a while, +knowing this isn't quite right, then you make up your mind +to try them anyhow--AND THEY WORK! Suddenly you're doing something +even your parents can't do. Six months ago you were just some kid--now, +you're the Crimson Flash of Area Code 512! You're bad--you're nationwide! + +Maybe you'll stop at a few abused codes. Maybe you'll decide that +boards aren't all that interesting after all, that it's wrong, +not worth the risk --but maybe you won't. The next step +is to pick up your own repeat-dialling program-- +to learn to generate your own stolen codes. +(This was dead easy five years ago, much harder +to get away with nowadays, but not yet impossible.) +And these dialling programs are not complex or intimidating-- +some are as small as twenty lines of software. + +Now, you too can share codes. You can trade codes to learn +other techniques. If you're smart enough to catch on, +and obsessive enough to want to bother, and ruthless enough +to start seriously bending rules, then you'll get better, fast. +You start to develop a rep. You move up to a heavier class +of board--a board with a bad attitude, the kind of board +that naive dopes like your classmates and your former self +have never even heard of! You pick up the jargon of phreaking +and hacking from the board. You read a few of those anarchy philes-- +and man, you never realized you could be a real OUTLAW without +ever leaving your bedroom. + +You still play other computer games, but now you have a new +and bigger game. This one will bring you a different kind of status +than destroying even eight zillion lousy space invaders. + +Hacking is perceived by hackers as a "game." This is +not an entirely unreasonable or sociopathic perception. +You can win or lose at hacking, succeed or fail, +but it never feels "real." It's not simply that +imaginative youngsters sometimes have a hard time +telling "make-believe" from "real life." Cyberspace +is NOT REAL! "Real" things are physical objects +like trees and shoes and cars. Hacking takes place +on a screen. Words aren't physical, numbers +(even telephone numbers and credit card numbers) +aren't physical. Sticks and stones may break my bones, +but data will never hurt me. Computers SIMULATE reality, +like computer games that simulate tank battles or dogfights +or spaceships. Simulations are just make-believe, +and the stuff in computers is NOT REAL. + +Consider this: if "hacking" is supposed to be so serious and +real-life and dangerous, then how come NINE-YEAR-OLD KIDS have +computers and modems? You wouldn't give a nine year old his own car, +or his own rifle, or his own chainsaw--those things are "real." + +People underground are perfectly aware that the "game" +is frowned upon by the powers that be. Word gets around +about busts in the underground. Publicizing busts is one +of the primary functions of pirate boards, but they also +promulgate an attitude about them, and their own idiosyncratic +ideas of justice. The users of underground boards won't complain +if some guy is busted for crashing systems, spreading viruses, +or stealing money by wire-fraud. They may shake their heads +with a sneaky grin, but they won't openly defend these practices. +But when a kid is charged with some theoretical amount of theft: +$233,846.14, for instance, because he sneaked into a computer +and copied something, and kept it in his house on a floppy disk-- +this is regarded as a sign of near-insanity from prosecutors, +a sign that they've drastically mistaken the immaterial game +of computing for their real and boring everyday world +of fatcat corporate money. + +It's as if big companies and their suck-up lawyers +think that computing belongs to them, and they can +retail it with price stickers, as if it were boxes +of laundry soap! But pricing "information" is like +trying to price air or price dreams. Well, anybody +on a pirate board knows that computing can be, +and ought to be, FREE. Pirate boards are little +independent worlds in cyberspace, and they don't belong +to anybody but the underground. Underground boards +aren't "brought to you by Procter & Gamble." + +To log on to an underground board can mean to +experience liberation, to enter a world where, +for once, money isn't everything and adults +don't have all the answers. + +Let's sample another vivid hacker manifesto. Here are +some excerpts from "The Conscience of a Hacker," by "The Mentor," +from Phrack Volume One, Issue 7, Phile 3. + +"I made a discovery today. I found a computer. +Wait a second, this is cool. It does what I want it to. +If it makes a mistake, it's because I screwed it up. +Not because it doesn't like me. (. . .) +"And then it happened. . .a door opened to a world. . . +rushing through the phone line like heroin through an +addict's veins, an electronic pulse is sent out, +a refuge from day-to-day incompetencies is sought. . . +a board is found. `This is it. . .this is where I belong. . .' +"I know everyone here. . .even if I've never met them, +never talked to them, may never hear from them again. . . +I know you all. . . (. . .) + +"This is our world now. . .the world of the electron +and the switch, the beauty of the baud. We make use of a +service already existing without paying for what could be +dirt-cheap if it wasn't run by profiteering gluttons, and you +call us criminals. We explore. . .and you call us criminals. +We seek after knowledge. . .and you call us criminals. +We exist without skin color, without nationality, +without religious bias. . .and you call us criminals. +You build atomic bombs, you wage wars, you murder, +cheat and lie to us and try to make us believe that +it's for our own good, yet we're the criminals. + +"Yes, I am a criminal. My crime is that of curiosity. +My crime is that of judging people by what they say and think, +not what they look like. My crime is that of outsmarting you, +something that you will never forgive me for." + +# + +There have been underground boards almost as long +as there have been boards. One of the first was 8BBS, +which became a stronghold of the West Coast phone-phreak elite. +After going on-line in March 1980, 8BBS sponsored "Susan Thunder," +and "Tuc," and, most notoriously, "the Condor." "The Condor" +bore the singular distinction of becoming the most vilified +American phreak and hacker ever. Angry underground associates, +fed up with Condor's peevish behavior, turned him in to police, +along with a heaping double-helping of outrageous hacker legendry. +As a result, Condor was kept in solitary confinement for seven months, +for fear that he might start World War Three by triggering missile silos +from the prison payphone. (Having served his time, Condor is now +walking around loose; WWIII has thus far conspicuously failed to occur.) + +The sysop of 8BBS was an ardent free-speech enthusiast +who simply felt that ANY attempt to restrict the expression +of his users was unconstitutional and immoral. +Swarms of the technically curious entered 8BBS +and emerged as phreaks and hackers, until, in 1982, +a friendly 8BBS alumnus passed the sysop a new modem +which had been purchased by credit-card fraud. +Police took this opportunity to seize the entire board +and remove what they considered an attractive nuisance. + +Plovernet was a powerful East Coast pirate board +that operated in both New York and Florida. +Owned and operated by teenage hacker "Quasi Moto," +Plovernet attracted five hundred eager users in 1983. +"Emmanuel Goldstein" was one-time co-sysop of Plovernet, +along with "Lex Luthor," founder of the "Legion of Doom" group. +Plovernet bore the signal honor of being the original home +of the "Legion of Doom," about which the reader will be hearing +a great deal, soon. + +"Pirate-80," or "P-80," run by a sysop known as "Scan-Man," +got into the game very early in Charleston, and continued +steadily for years. P-80 flourished so flagrantly that +even its most hardened users became nervous, and some +slanderously speculated that "Scan Man" must have ties +to corporate security, a charge he vigorously denied. + +"414 Private" was the home board for the first GROUP +to attract conspicuous trouble, the teenage "414 Gang," +whose intrusions into Sloan-Kettering Cancer Center and +Los Alamos military computers were to be a nine-days-wonder in 1982. + +At about this time, the first software piracy boards +began to open up, trading cracked games for the Atari 800 +and the Commodore C64. Naturally these boards were +heavily frequented by teenagers. And with the 1983 +release of the hacker-thriller movie War Games, +the scene exploded. It seemed that every kid +in America had demanded and gotten a modem for Christmas. +Most of these dabbler wannabes put their modems in the attic +after a few weeks, and most of the remainder minded their +P's and Q's and stayed well out of hot water. But some +stubborn and talented diehards had this hacker kid in +War Games figured for a happening dude. They simply +could not rest until they had contacted the underground-- +or, failing that, created their own. + +In the mid-80s, underground boards sprang up like digital fungi. +ShadowSpawn Elite. Sherwood Forest I, II, and III. +Digital Logic Data Service in Florida, sysoped by no less +a man than "Digital Logic" himself; Lex Luthor of the +Legion of Doom was prominent on this board, since it +was in his area code. Lex's own board, "Legion of Doom," +started in 1984. The Neon Knights ran a network of Apple- +hacker boards: Neon Knights North, South, East and West. +Free World II was run by "Major Havoc." Lunatic Labs +is still in operation as of this writing. Dr. Ripco +in Chicago, an anything-goes anarchist board with an +extensive and raucous history, was seized by Secret Service +agents in 1990 on Sundevil day, but up again almost immediately, +with new machines and scarcely diminished vigor. + +The St. Louis scene was not to rank with major centers +of American hacking such as New York and L.A. But St. +Louis did rejoice in possession of "Knight Lightning" +and "Taran King," two of the foremost JOURNALISTS native +to the underground. Missouri boards like Metal Shop, +Metal Shop Private, Metal Shop Brewery, may not have +been the heaviest boards around in terms of illicit +expertise. But they became boards where hackers could +exchange social gossip and try to figure out what the +heck was going on nationally--and internationally. +Gossip from Metal Shop was put into the form of news files, +then assembled into a general electronic publication, +Phrack, a portmanteau title coined from "phreak" and "hack." +The Phrack editors were as obsessively curious about other +hackers as hackers were about machines. + +Phrack, being free of charge and lively reading, began +to circulate throughout the underground. As Taran King +and Knight Lightning left high school for college, +Phrack began to appear on mainframe machines linked to BITNET, +and, through BITNET to the "Internet," that loose but +extremely potent not-for-profit network where academic, +governmental and corporate machines trade data through +the UNIX TCP/IP protocol. (The "Internet Worm" of +November 2-3,1988, created by Cornell grad student Robert Morris, +was to be the largest and best-publicized computer-intrusion scandal +to date. Morris claimed that his ingenious "worm" program was meant +to harmlessly explore the Internet, but due to bad programming, +the Worm replicated out of control and crashed some six thousand +Internet computers. Smaller-scale and less ambitious Internet hacking +was a standard for the underground elite.) + +Most any underground board not hopelessly lame and out-of-it +would feature a complete run of Phrack--and, possibly, +the lesser-known standards of the underground: +the Legion of Doom Technical Journal, the obscene +and raucous Cult of the Dead Cow files, P/HUN magazine, +Pirate, the Syndicate Reports, and perhaps the highly +anarcho-political Activist Times Incorporated. + +Possession of Phrack on one's board was prima facie +evidence of a bad attitude. Phrack was seemingly everywhere, +aiding, abetting, and spreading the underground ethos. +And this did not escape the attention of corporate security +or the police. + +We now come to the touchy subject of police and boards. +Police, do, in fact, own boards. In 1989, there were +police-sponsored boards in California, Colorado, Florida, +Georgia, Idaho, Michigan, Missouri, Texas, and Virginia: +boards such as "Crime Bytes," "Crimestoppers," "All Points" +and "Bullet-N-Board." Police officers, as private computer +enthusiasts, ran their own boards in Arizona, California, +Colorado, Connecticut, Florida, Missouri, Maryland, +New Mexico, North Carolina, Ohio, Tennessee and Texas. +Police boards have often proved helpful in community relations. +Sometimes crimes are reported on police boards. + +Sometimes crimes are COMMITTED on police boards. +This has sometimes happened by accident, as naive hackers +blunder onto police boards and blithely begin offering telephone codes. +Far more often, however, it occurs through the now almost-traditional +use of "sting boards." The first police sting-boards were established +in 1985: "Underground Tunnel" in Austin, Texas, whose sysop +Sgt. Robert Ansley called himself "Pluto"--"The Phone Company" +in Phoenix, Arizona, run by Ken MacLeod of the Maricopa County +Sheriff's office--and Sgt. Dan Pasquale's board in Fremont, California. +Sysops posed as hackers, and swiftly garnered coteries of ardent users, +who posted codes and loaded pirate software with abandon, +and came to a sticky end. + +Sting boards, like other boards, are cheap to operate, +very cheap by the standards of undercover police operations. +Once accepted by the local underground, sysops will likely be +invited into other pirate boards, where they can compile more dossiers. +And when the sting is announced and the worst offenders arrested, +the publicity is generally gratifying. The resultant paranoia +in the underground--perhaps more justly described as a "deterrence effect"-- +tends to quell local lawbreaking for quite a while. + +Obviously police do not have to beat the underbrush for hackers. +On the contrary, they can go trolling for them. Those caught +can be grilled. Some become useful informants. They can lead +the way to pirate boards all across the country. + +And boards all across the country showed the sticky +fingerprints of Phrack, and of that loudest and most +flagrant of all underground groups, the "Legion of Doom." + +The term "Legion of Doom" came from comic books. The Legion of Doom, +a conspiracy of costumed super- villains headed by the chrome-domed +criminal ultra- mastermind Lex Luthor, gave Superman a lot of four-color +graphic trouble for a number of decades. Of course, Superman, +that exemplar of Truth, Justice, and the American Way, +always won in the long run. This didn't matter to the hacker Doomsters-- +"Legion of Doom" was not some thunderous and evil Satanic reference, +it was not meant to be taken seriously. "Legion of Doom" came +from funny-books and was supposed to be funny. + +"Legion of Doom" did have a good mouthfilling ring to it, though. +It sounded really cool. Other groups, such as the "Farmers of Doom," +closely allied to LoD, recognized this grandiloquent quality, +and made fun of it. There was even a hacker group called +"Justice League of America," named after Superman's club +of true-blue crimefighting superheros. + +But they didn't last; the Legion did. + +The original Legion of Doom, hanging out on Quasi Moto's Plovernet board, +were phone phreaks. They weren't much into computers. "Lex Luthor" himself +(who was under eighteen when he formed the Legion) was a COSMOS expert, +COSMOS being the "Central System for Mainframe Operations," +a telco internal computer network. Lex would eventually become +quite a dab hand at breaking into IBM mainframes, but although +everyone liked Lex and admired his attitude, he was not considered +a truly accomplished computer intruder. Nor was he the "mastermind" +of the Legion of Doom--LoD were never big on formal leadership. +As a regular on Plovernet and sysop of his "Legion of Doom BBS," +Lex was the Legion's cheerleader and recruiting officer. + +Legion of Doom began on the ruins of an earlier phreak group, +The Knights of Shadow. Later, LoD was to subsume the personnel +of the hacker group "Tribunal of Knowledge." People came and went +constantly in LoD; groups split up or formed offshoots. + +Early on, the LoD phreaks befriended a few computer-intrusion +enthusiasts, who became the associated "Legion of Hackers." +Then the two groups conflated into the "Legion of Doom/Hackers," +or LoD/H. When the original "hacker" wing, Messrs. "Compu-Phreak" +and "Phucked Agent 04," found other matters to occupy their time, +the extra "/H" slowly atrophied out of the name; but by this time +the phreak wing, Messrs. Lex Luthor, "Blue Archer," "Gary Seven," +"Kerrang Khan," "Master of Impact," "Silver Spy," "The Marauder," +and "The Videosmith," had picked up a plethora of intrusion +expertise and had become a force to be reckoned with. + +LoD members seemed to have an instinctive understanding +that the way to real power in the underground lay through +covert publicity. LoD were flagrant. Not only was it one +of the earliest groups, but the members took pains to widely +distribute their illicit knowledge. Some LoD members, +like "The Mentor," were close to evangelical about it. +Legion of Doom Technical Journal began to show up on boards +throughout the underground. + +LoD Technical Journal was named in cruel parody +of the ancient and honored AT&T Technical Journal. +The material in these two publications was quite similar-- +much of it, adopted from public journals and discussions +in the telco community. And yet, the predatory attitude +of LoD made even its most innocuous data seem deeply sinister; +an outrage; a clear and present danger. + +To see why this should be, let's consider the following +(invented) paragraphs, as a kind of thought experiment. + +(A) "W. Fred Brown, AT&T Vice President for +Advanced Technical Development, testified May 8 +at a Washington hearing of the National Telecommunications +and Information Administration (NTIA), regarding +Bellcore's GARDEN project. GARDEN (Generalized +Automatic Remote Distributed Electronic Network) is a +telephone-switch programming tool that makes it possible +to develop new telecom services, including hold-on-hold +and customized message transfers, from any keypad terminal, +within seconds. The GARDEN prototype combines centrex +lines with a minicomputer using UNIX operating system software." + +(B) "Crimson Flash 512 of the Centrex Mobsters reports: +D00dz, you wouldn't believe this GARDEN bullshit Bellcore's +just come up with! Now you don't even need a lousy Commodore +to reprogram a switch--just log on to GARDEN as a technician, +and you can reprogram switches right off the keypad in any +public phone booth! You can give yourself hold-on-hold +and customized message transfers, and best of all, +the thing is run off (notoriously insecure) centrex lines +using--get this--standard UNIX software! Ha ha ha ha!" + +Message (A), couched in typical techno-bureaucratese, +appears tedious and almost unreadable. (A) scarcely seems +threatening or menacing. Message (B), on the other hand, +is a dreadful thing, prima facie evidence of a dire conspiracy, +definitely not the kind of thing you want your teenager reading. + +The INFORMATION, however, is identical. It is PUBLIC +information, presented before the federal government in +an open hearing. It is not "secret." It is not "proprietary." +It is not even "confidential." On the contrary, the +development of advanced software systems is a matter +of great public pride to Bellcore. + +However, when Bellcore publicly announces a project of this kind, +it expects a certain attitude from the public--something along +the lines of GOSH WOW, YOU GUYS ARE GREAT, KEEP THAT UP, WHATEVER IT IS-- +certainly not cruel mimickry, one-upmanship and outrageous speculations +about possible security holes. + +Now put yourself in the place of a policeman confronted by +an outraged parent, or telco official, with a copy of Version (B). +This well-meaning citizen, to his horror, has discovered +a local bulletin-board carrying outrageous stuff like (B), +which his son is examining with a deep and unhealthy interest. +If (B) were printed in a book or magazine, you, as an American +law enforcement officer, would know that it would take +a hell of a lot of trouble to do anything about it; +but it doesn't take technical genius to recognize that +if there's a computer in your area harboring stuff like (B), +there's going to be trouble. + +In fact, if you ask around, any computer-literate cop +will tell you straight out that boards with stuff like (B) +are the SOURCE of trouble. And the WORST source of trouble +on boards are the ringleaders inventing and spreading stuff like (B). +If it weren't for these jokers, there wouldn't BE any trouble. + +And Legion of Doom were on boards like nobody else. +Plovernet. The Legion of Doom Board. The Farmers of Doom Board. +Metal Shop. OSUNY. Blottoland. Private Sector. Atlantis. +Digital Logic. Hell Phrozen Over. + +LoD members also ran their own boards. "Silver Spy" started +his own board, "Catch-22," considered one of the heaviest around. +So did "Mentor," with his "Phoenix Project." When they didn't run boards +themselves, they showed up on other people's boards, to brag, boast, +and strut. And where they themselves didn't go, their philes went, +carrying evil knowledge and an even more evil attitude. + +As early as 1986, the police were under the vague impression +that EVERYONE in the underground was Legion of Doom. +LoD was never that large--considerably smaller than either +"Metal Communications" or "The Administration," for instance-- +but LoD got tremendous press. Especially in Phrack, +which at times read like an LoD fan magazine; and Phrack +was everywhere, especially in the offices of telco security. +You couldn't GET busted as a phone phreak, a hacker, +or even a lousy codes kid or warez dood, without the cops +asking if you were LoD. + +This was a difficult charge to deny, as LoD never +distributed membership badges or laminated ID cards. +If they had, they would likely have died out quickly, +for turnover in their membership was considerable. +LoD was less a high-tech street-gang than an ongoing +state-of-mind. LoD was the Gang That Refused to Die. +By 1990, LoD had RULED for ten years, and it seemed WEIRD +to police that they were continually busting people who were +only sixteen years old. All these teenage small-timers +were pleading the tiresome hacker litany of "just curious, +no criminal intent." Somewhere at the center of this +conspiracy there had to be some serious adult masterminds, +not this seemingly endless supply of myopic suburban +white kids with high SATs and funny haircuts. + +There was no question that most any American hacker +arrested would "know" LoD. They knew the handles +of contributors to LoD Tech Journal, and were likely +to have learned their craft through LoD boards and LoD activism. +But they'd never met anyone from LoD. Even some of the +rotating cadre who were actually and formally "in LoD" +knew one another only by board-mail and pseudonyms. +This was a highly unconventional profile for a criminal conspiracy. +Computer networking, and the rapid evolution of the digital underground, +made the situation very diffuse and confusing. + +Furthermore, a big reputation in the digital underground +did not coincide with one's willingness to commit "crimes." +Instead, reputation was based on cleverness and technical mastery. +As a result, it often seemed that the HEAVIER the hackers were, +the LESS likely they were to have committed any kind of common, +easily prosecutable crime. There were some hackers who could really steal. +And there were hackers who could really hack. But the two groups didn't seem +to overlap much, if at all. For instance, most people in the underground +looked up to "Emmanuel Goldstein" of 2600 as a hacker demigod. +But Goldstein's publishing activities were entirely legal-- +Goldstein just printed dodgy stuff and talked about politics, +he didn't even hack. When you came right down to it, +Goldstein spent half his time complaining that computer security +WASN'T STRONG ENOUGH and ought to be drastically improved +across the board! + +Truly heavy-duty hackers, those with serious technical skills +who had earned the respect of the underground, never stole money +or abused credit cards. Sometimes they might abuse phone-codes-- +but often, they seemed to get all the free phone-time they wanted +without leaving a trace of any kind. + +The best hackers, the most powerful and technically accomplished, +were not professional fraudsters. They raided computers habitually, +but wouldn't alter anything, or damage anything. They didn't even steal +computer equipment--most had day-jobs messing with hardware, +and could get all the cheap secondhand equipment they wanted. +The hottest hackers, unlike the teenage wannabes, weren't snobs +about fancy or expensive hardware. Their machines tended to be +raw second-hand digital hot-rods full of custom add-ons that +they'd cobbled together out of chickenwire, memory chips and spit. +Some were adults, computer software writers and consultants by trade, +and making quite good livings at it. Some of them ACTUALLY WORKED +FOR THE PHONE COMPANY--and for those, the "hackers" actually found +under the skirts of Ma Bell, there would be little mercy in 1990. + +It has long been an article of faith in the +underground that the "best" hackers never get caught. +They're far too smart, supposedly. They never get caught +because they never boast, brag, or strut. These demigods +may read underground boards (with a condescending smile), +but they never say anything there. The "best" hackers, +according to legend, are adult computer professionals, +such as mainframe system administrators, who already know +the ins and outs of their particular brand of security. +Even the "best" hacker can't break in to just any computer at random: +the knowledge of security holes is too specialized, varying widely +with different software and hardware. But if people are employed to run, +say, a UNIX mainframe or a VAX/VMS machine, then they tend to learn +security from the inside out. Armed with this knowledge, +they can look into most anybody else's UNIX or VMS +without much trouble or risk, if they want to. +And, according to hacker legend, of course they want to, +so of course they do. They just don't make a big deal +of what they've done. So nobody ever finds out. + +It is also an article of faith in the underground that +professional telco people "phreak" like crazed weasels. +OF COURSE they spy on Madonna's phone calls--I mean, +WOULDN'T YOU? Of course they give themselves free long- +distance--why the hell should THEY pay, they're running +the whole shebang! + +It has, as a third matter, long been an article of faith +that any hacker caught can escape serious punishment if +he confesses HOW HE DID IT. Hackers seem to believe +that governmental agencies and large corporations are +blundering about in cyberspace like eyeless jellyfish +or cave salamanders. They feel that these large +but pathetically stupid organizations will proffer up +genuine gratitude, and perhaps even a security post +and a big salary, to the hot-shot intruder who will deign +to reveal to them the supreme genius of his modus operandi. + +In the case of longtime LoD member "Control-C," +this actually happened, more or less. Control-C had led +Michigan Bell a merry chase, and when captured in 1987, +he turned out to be a bright and apparently physically +harmless young fanatic, fascinated by phones. There was +no chance in hell that Control-C would actually repay the +enormous and largely theoretical sums in long-distance +service that he had accumulated from Michigan Bell. +He could always be indicted for fraud or computer-intrusion, +but there seemed little real point in this--he hadn't +physically damaged any computer. He'd just plead guilty, +and he'd likely get the usual slap-on-the-wrist, +and in the meantime it would be a big hassle for Michigan Bell +just to bring up the case. But if kept on the payroll, +he might at least keep his fellow hackers at bay. + +There were uses for him. For instance, a contrite +Control-C was featured on Michigan Bell internal posters, +sternly warning employees to shred their trash. +He'd always gotten most of his best inside info from +"trashing"--raiding telco dumpsters, for useful data +indiscreetly thrown away. He signed these posters, too. +Control-C had become something like a Michigan Bell mascot. +And in fact, Control-C DID keep other hackers at bay. +Little hackers were quite scared of Control-C and his +heavy-duty Legion of Doom friends. And big hackers WERE +his friends and didn't want to screw up his cushy situation. + +No matter what one might say of LoD, they did stick together. +When "Wasp," an apparently genuinely malicious New York hacker, +began crashing Bellcore machines, Control-C received swift volunteer +help from "the Mentor" and the Georgia LoD wing made up of +"The Prophet," "Urvile," and "Leftist." Using Mentor's Phoenix +Project board to coordinate, the Doomsters helped telco security +to trap Wasp, by luring him into a machine with a tap +and line-trace installed. Wasp lost. LoD won! And my, did they brag. + +Urvile, Prophet and Leftist were well-qualified for this activity, +probably more so even than the quite accomplished Control-C. +The Georgia boys knew all about phone switching-stations. +Though relative johnny-come-latelies in the Legion of Doom, +they were considered some of LoD's heaviest guys, +into the hairiest systems around. They had the good fortune +to live in or near Atlanta, home of the sleepy and apparently +tolerant BellSouth RBOC. + +As RBOC security went, BellSouth were "cake." US West (of Arizona, +the Rockies and the Pacific Northwest) were tough and aggressive, +probably the heaviest RBOC around. Pacific Bell, California's PacBell, +were sleek, high-tech, and longtime veterans of the LA phone-phreak wars. +NYNEX had the misfortune to run the New York City area, and were warily +prepared for most anything. Even Michigan Bell, a division of the +Ameritech RBOC, at least had the elementary sense to hire their own hacker +as a useful scarecrow. But BellSouth, even though their corporate P.R. +proclaimed them to have "Everything You Expect From a Leader," were pathetic. + +When rumor about LoD's mastery of Georgia's switching network got around +to BellSouth through Bellcore and telco security scuttlebutt, +they at first refused to believe it. If you paid serious attention +to every rumor out and about these hacker kids, you would hear all kinds +of wacko saucer-nut nonsense: that the National Security Agency +monitored all American phone calls, that the CIA and DEA tracked +traffic on bulletin-boards with word-analysis programs, +that the Condor could start World War III from a payphone. + +If there were hackers into BellSouth switching-stations, then how come +nothing had happened? Nothing had been hurt. BellSouth's machines +weren't crashing. BellSouth wasn't suffering especially badly from fraud. +BellSouth's customers weren't complaining. BellSouth was headquartered +in Atlanta, ambitious metropolis of the new high-tech Sunbelt; +and BellSouth was upgrading its network by leaps and bounds, +digitizing the works left right and center. They could hardly be +considered sluggish or naive. BellSouth's technical expertise +was second to none, thank you kindly. But then came the Florida business. + +On June 13, 1989, callers to the Palm Beach County Probation Department, +in Delray Beach, Florida, found themselves involved in a remarkable +discussion with a phone-sex worker named "Tina" in New York State. +Somehow, ANY call to this probation office near Miami was instantly +and magically transported across state lines, at no extra charge to the user, +to a pornographic phone-sex hotline hundreds of miles away! + +This practical joke may seem utterly hilarious at first hearing, +and indeed there was a good deal of chuckling about it in +phone phreak circles, including the Autumn 1989 issue of 2600. +But for Southern Bell (the division of the BellSouth RBOC +supplying local service for Florida, Georgia, North Carolina +and South Carolina), this was a smoking gun. For the first time ever, +a computer intruder had broken into a BellSouth central office +switching station and re-programmed it! + +Or so BellSouth thought in June 1989. Actually, LoD members had been +frolicking harmlessly in BellSouth switches since September 1987. +The stunt of June 13--call-forwarding a number through manipulation +of a switching station--was child's play for hackers as accomplished +as the Georgia wing of LoD. Switching calls interstate sounded like +a big deal, but it took only four lines of code to accomplish this. +An easy, yet more discreet, stunt, would be to call-forward another +number to your own house. If you were careful and considerate, +and changed the software back later, then not a soul would know. +Except you. And whoever you had bragged to about it. + +As for BellSouth, what they didn't know wouldn't hurt them. + +Except now somebody had blown the whole thing wide open, and BellSouth knew. + +A now alerted and considerably paranoid BellSouth began searching switches +right and left for signs of impropriety, in that hot summer of 1989. +No fewer than forty-two BellSouth employees were put on 12-hour shifts, +twenty-four hours a day, for two solid months, poring over records +and monitoring computers for any sign of phony access. These forty-two +overworked experts were known as BellSouth's "Intrusion Task Force." + +What the investigators found astounded them. Proprietary telco databases +had been manipulated: phone numbers had been created out of thin air, +with no users' names and no addresses. And perhaps worst of all, +no charges and no records of use. The new digital ReMOB (Remote Observation) +diagnostic feature had been extensively tampered with--hackers had learned to +reprogram ReMOB software, so that they could listen in on any switch-routed +call at their leisure! They were using telco property to SPY! + +The electrifying news went out throughout law enforcement in 1989. +It had never really occurred to anyone at BellSouth that their prized +and brand-new digital switching-stations could be RE-PROGRAMMED. +People seemed utterly amazed that anyone could have the nerve. +Of course these switching stations were "computers," and everybody +knew hackers liked to "break into computers:" but telephone people's +computers were DIFFERENT from normal people's computers. + +The exact reason WHY these computers were "different" was +rather ill-defined. It certainly wasn't the extent of their security. +The security on these BellSouth computers was lousy; the AIMSX computers, +for instance, didn't even have passwords. But there was no question that +BellSouth strongly FELT that their computers were very different indeed. +And if there were some criminals out there who had not gotten that message, +BellSouth was determined to see that message taught. + +After all, a 5ESS switching station was no mere bookkeeping system for +some local chain of florists. Public service depended on these stations. +Public SAFETY depended on these stations. + +And hackers, lurking in there call-forwarding or ReMobbing, could spy +on anybody in the local area! They could spy on telco officials! +They could spy on police stations! They could spy on local offices +of the Secret Service. . . . + +In 1989, electronic cops and hacker-trackers began using scrambler-phones +and secured lines. It only made sense. There was no telling who was into +those systems. Whoever they were, they sounded scary. This was some +new level of antisocial daring. Could be West German hackers, in the pay +of the KGB. That too had seemed a weird and farfetched notion, +until Clifford Stoll had poked and prodded a sluggish Washington +law-enforcement bureaucracy into investigating a computer intrusion +that turned out to be exactly that--HACKERS, IN THE PAY OF THE KGB! +Stoll, the systems manager for an Internet lab in Berkeley California, +had ended up on the front page of the New Nork Times, proclaimed a national +hero in the first true story of international computer espionage. +Stoll's counterspy efforts, which he related in a bestselling book, +The Cuckoo's Egg, in 1989, had established the credibility of `hacking' +as a possible threat to national security. The United States Secret Service +doesn't mess around when it suspects a possible action by a foreign +intelligence apparat. + +The Secret Service scrambler-phones and secured lines put +a tremendous kink in law enforcement's ability to operate freely; +to get the word out, cooperate, prevent misunderstandings. +Nevertheless, 1989 scarcely seemed the time for half-measures. +If the police and Secret Service themselves were not operationally secure, +then how could they reasonably demand measures of security from +private enterprise? At least, the inconvenience made people aware +of the seriousness of the threat. + +If there was a final spur needed to get the police off the dime, +it came in the realization that the emergency 911 system was vulnerable. +The 911 system has its own specialized software, but it is run on the same +digital switching systems as the rest of the telephone network. +911 is not physically different from normal telephony. But it is +certainly culturally different, because this is the area of +telephonic cyberspace reserved for the police and emergency services. + +Your average policeman may not know much about hackers or phone-phreaks. +Computer people are weird; even computer COPS are rather weird; +the stuff they do is hard to figure out. But a threat to the 911 system +is anything but an abstract threat. If the 911 system goes, people can die. + +Imagine being in a car-wreck, staggering to a phone-booth, +punching 911 and hearing "Tina" pick up the phone-sex line +somewhere in New York! The situation's no longer comical, somehow. + +And was it possible? No question. Hackers had attacked 911 +systems before. Phreaks can max-out 911 systems just by siccing +a bunch of computer-modems on them in tandem, dialling them over +and over until they clog. That's very crude and low-tech, +but it's still a serious business. + +The time had come for action. It was time to take stern measures +with the underground. It was time to start picking up the dropped threads, +the loose edges, the bits of braggadocio here and there; it was time to get +on the stick and start putting serious casework together. Hackers weren't +"invisible." They THOUGHT they were invisible; but the truth was, +they had just been tolerated too long. + +Under sustained police attention in the summer of '89, the digital +underground began to unravel as never before. + +The first big break in the case came very early on: July 1989, +the following month. The perpetrator of the "Tina" switch was caught, +and confessed. His name was "Fry Guy," a 16-year-old in Indiana. +Fry Guy had been a very wicked young man. + +Fry Guy had earned his handle from a stunt involving French fries. +Fry Guy had filched the log-in of a local MacDonald's manager +and had logged-on to the MacDonald's mainframe on the Sprint +Telenet system. Posing as the manager, Fry Guy had altered +MacDonald's records, and given some teenage hamburger-flipping +friends of his, generous raises. He had not been caught. + +Emboldened by success, Fry Guy moved on to credit-card abuse. +Fry Guy was quite an accomplished talker; with a gift for +"social engineering." If you can do "social engineering" +--fast-talk, fake-outs, impersonation, conning, scamming-- +then card abuse comes easy. (Getting away with it in +the long run is another question). + +Fry Guy had run across "Urvile" of the Legion of Doom +on the ALTOS Chat board in Bonn, Germany. ALTOS Chat +was a sophisticated board, accessible through globe-spanning +computer networks like BITnet, Tymnet, and Telenet. +ALTOS was much frequented by members of Germany's +Chaos Computer Club. Two Chaos hackers who hung out on ALTOS, +"Jaeger" and "Pengo," had been the central villains of +Clifford Stoll's Cuckoo's Egg case: consorting in East Berlin +with a spymaster from the KGB, and breaking into American +computers for hire, through the Internet. + +When LoD members learned the story of Jaeger's depredations +from Stoll's book, they were rather less than impressed, +technically speaking. On LoD's own favorite board of the moment, +"Black Ice," LoD members bragged that they themselves could have done +all the Chaos break-ins in a week flat! Nevertheless, LoD were grudgingly +impressed by the Chaos rep, the sheer hairy-eyed daring of hash-smoking +anarchist hackers who had rubbed shoulders with the fearsome big-boys +of international Communist espionage. LoD members sometimes traded +bits of knowledge with friendly German hackers on ALTOS--phone numbers +for vulnerable VAX/VMS computers in Georgia, for instance. +Dutch and British phone phreaks, and the Australian clique of +"Phoenix," "Nom," and "Electron," were ALTOS regulars, too. +In underground circles, to hang out on ALTOS was considered +the sign of an elite dude, a sophisticated hacker of the +international digital jet-set. + +Fry Guy quickly learned how to raid information from credit-card +consumer-reporting agencies. He had over a hundred stolen credit-card +numbers in his notebooks, and upwards of a thousand swiped long-distance +access codes. He knew how to get onto Altos, and how to talk the talk of +the underground convincingly. He now wheedled knowledge of switching-station +tricks from Urvile on the ALTOS system. + +Combining these two forms of knowledge enabled Fry Guy to bootstrap +his way up to a new form of wire-fraud. First, he'd snitched credit card +numbers from credit-company computers. The data he copied included names, +addresses and phone numbers of the random card-holders. + +Then Fry Guy, impersonating a card-holder, called up Western Union +and asked for a cash advance on "his" credit card. Western Union, +as a security guarantee, would call the customer back, at home, +to verify the transaction. + +But, just as he had switched the Florida probation office to "Tina" +in New York, Fry Guy switched the card-holder's number to a local pay-phone. +There he would lurk in wait, muddying his trail by routing and re-routing +the call, through switches as far away as Canada. When the call came through, +he would boldly "social-engineer," or con, the Western Union people, pretending +to be the legitimate card-holder. Since he'd answered the proper phone number, +the deception was not very hard. Western Union's money was then shipped to +a confederate of Fry Guy's in his home town in Indiana. + +Fry Guy and his cohort, using LoD techniques, stole six thousand dollars +from Western Union between December 1988 and July 1989. They also dabbled +in ordering delivery of stolen goods through card-fraud. Fry Guy +was intoxicated with success. The sixteen-year-old fantasized wildly +to hacker rivals, boasting that he'd used rip-off money to hire himself +a big limousine, and had driven out-of-state with a groupie from +his favorite heavy-metal band, Motley Crue. + +Armed with knowledge, power, and a gratifying stream of free money, +Fry Guy now took it upon himself to call local representatives +of Indiana Bell security, to brag, boast, strut, and utter +tormenting warnings that his powerful friends in the notorious +Legion of Doom could crash the national telephone network. +Fry Guy even named a date for the scheme: the Fourth of July, +a national holiday. + +This egregious example of the begging-for-arrest syndrome was shortly +followed by Fry Guy's arrest. After the Indiana telephone company figured +out who he was, the Secret Service had DNRs--Dialed Number Recorders-- +installed on his home phone lines. These devices are not taps, and can't +record the substance of phone calls, but they do record the phone numbers +of all calls going in and out. Tracing these numbers showed Fry Guy's +long-distance code fraud, his extensive ties to pirate bulletin boards, +and numerous personal calls to his LoD friends in Atlanta. By July 11, +1989, Prophet, Urvile and Leftist also had Secret Service DNR +"pen registers" installed on their own lines. + +The Secret Service showed up in force at Fry Guy's house on July 22, 1989, +to the horror of his unsuspecting parents. The raiders were led by +a special agent from the Secret Service's Indianapolis office. +However, the raiders were accompanied and advised by Timothy M. Foley +of the Secret Service's Chicago office (a gentleman about whom +we will soon be hearing a great deal). + +Following federal computer-crime techniques that had been standard +since the early 1980s, the Secret Service searched the house thoroughly, +and seized all of Fry Guy's electronic equipment and notebooks. +All Fry Guy's equipment went out the door in the custody of the +Secret Service, which put a swift end to his depredations. + +The USSS interrogated Fry Guy at length. His case was put in the charge +of Deborah Daniels, the federal US Attorney for the Southern District +of Indiana. Fry Guy was charged with eleven counts of computer fraud, +unauthorized computer access, and wire fraud. The evidence was thorough +and irrefutable. For his part, Fry Guy blamed his corruption on the +Legion of Doom and offered to testify against them. + +Fry Guy insisted that the Legion intended to crash the phone system +on a national holiday. And when AT&T crashed on Martin Luther King Day, +1990, this lent a credence to his claim that genuinely alarmed telco +security and the Secret Service. + +Fry Guy eventually pled guilty on May 31, 1990. On September 14, +he was sentenced to forty-four months' probation and four hundred hours' +community service. He could have had it much worse; but it made sense +to prosecutors to take it easy on this teenage minor, while zeroing +in on the notorious kingpins of the Legion of Doom. + +But the case against LoD had nagging flaws. Despite the best effort +of investigators, it was impossible to prove that the Legion had crashed +the phone system on January 15, because they, in fact, hadn't done so. +The investigations of 1989 did show that certain members of +the Legion of Doom had achieved unprecedented power over the telco +switching stations, and that they were in active conspiracy +to obtain more power yet. Investigators were privately convinced +that the Legion of Doom intended to do awful things with this knowledge, +but mere evil intent was not enough to put them in jail. + +And although the Atlanta Three--Prophet, Leftist, and especially Urvile-- +had taught Fry Guy plenty, they were not themselves credit-card fraudsters. +The only thing they'd "stolen" was long-distance service--and since they'd +done much of that through phone-switch manipulation, there was no easy way +to judge how much they'd "stolen," or whether this practice was even "theft" +of any easily recognizable kind. + +Fry Guy's theft of long-distance codes had cost the phone companies plenty. +The theft of long-distance service may be a fairly theoretical "loss," +but it costs genuine money and genuine time to delete all those stolen codes, +and to re-issue new codes to the innocent owners of those corrupted codes. +The owners of the codes themselves are victimized, and lose time and money +and peace of mind in the hassle. And then there were the credit-card victims +to deal with, too, and Western Union. When it came to rip-off, Fry Guy was +far more of a thief than LoD. It was only when it came to actual computer +expertise that Fry Guy was small potatoes. + +The Atlanta Legion thought most "rules" of cyberspace were for rodents +and losers, but they DID have rules. THEY NEVER CRASHED ANYTHING, +AND THEY NEVER TOOK MONEY. These were rough rules-of-thumb, and +rather dubious principles when it comes to the ethical subtleties +of cyberspace, but they enabled the Atlanta Three to operate with +a relatively clear conscience (though never with peace of mind). + +If you didn't hack for money, if you weren't robbing people of actual funds +--money in the bank, that is-- then nobody REALLY got hurt, in LoD's opinion. +"Theft of service" was a bogus issue, and "intellectual property" was +a bad joke. But LoD had only elitist contempt for rip-off artists, +"leechers," thieves. They considered themselves clean. In their opinion, +if you didn't smash-up or crash any systems --(well, not on purpose, anyhow-- +accidents can happen, just ask Robert Morris) then it was very unfair +to call you a "vandal" or a "cracker." When you were hanging out on-line +with your "pals" in telco security, you could face them down from the higher +plane of hacker morality. And you could mock the police from the supercilious +heights of your hacker's quest for pure knowledge. + +But from the point of view of law enforcement and telco security, however, +Fry Guy was not really dangerous. The Atlanta Three WERE dangerous. +It wasn't the crimes they were committing, but the DANGER, +the potential hazard, the sheer TECHNICAL POWER LoD had accumulated, +that had made the situation untenable. Fry Guy was not LoD. +He'd never laid eyes on anyone in LoD; his only contacts with them +had been electronic. Core members of the Legion of Doom tended to meet +physically for conventions every year or so, to get drunk, give each other +the hacker high-sign, send out for pizza and ravage hotel suites. +Fry Guy had never done any of this. Deborah Daniels assessed Fry Guy +accurately as "an LoD wannabe." + +Nevertheless Fry Guy's crimes would be directly attributed to LoD +in much future police propaganda. LoD would be described as +"a closely knit group" involved in "numerous illegal activities" +including "stealing and modifying individual credit histories," +and "fraudulently obtaining money and property." Fry Guy did this, +but the Atlanta Three didn't; they simply weren't into theft, +but rather intrusion. This caused a strange kink in +the prosecution's strategy. LoD were accused of +"disseminating information about attacking computers +to other computer hackers in an effort to shift the focus +of law enforcement to those other hackers and away from the Legion of Doom." + +This last accusation (taken directly from a press release by the Chicago +Computer Fraud and Abuse Task Force) sounds particularly far-fetched. +One might conclude at this point that investigators would have been +well-advised to go ahead and "shift their focus" from the "Legion of Doom." +Maybe they SHOULD concentrate on "those other hackers"--the ones who were +actually stealing money and physical objects. + +But the Hacker Crackdown of 1990 was not a simple policing action. +It wasn't meant just to walk the beat in cyberspace--it was a CRACKDOWN, +a deliberate attempt to nail the core of the operation, to send a dire +and potent message that would settle the hash of the digital underground +for good. + +By this reasoning, Fry Guy wasn't much more than the electronic equivalent +of a cheap streetcorner dope dealer. As long as the masterminds of LoD were +still flagrantly operating, pushing their mountains of illicit knowledge +right and left, and whipping up enthusiasm for blatant lawbreaking, +then there would be an INFINITE SUPPLY of Fry Guys. + +Because LoD were flagrant, they had left trails everywhere, +to be picked up by law enforcement in New York, Indiana, +Florida, Texas, Arizona, Missouri, even Australia. +But 1990's war on the Legion of Doom was led out of Illinois, +by the Chicago Computer Fraud and Abuse Task Force. + +# + +The Computer Fraud and Abuse Task Force, led by federal prosecutor +William J. Cook, had started in 1987 and had swiftly become one +of the most aggressive local "dedicated computer-crime units." +Chicago was a natural home for such a group. The world's first +computer bulletin-board system had been invented in Illinois. +The state of Illinois had some of the nation's first and sternest +computer crime laws. Illinois State Police were markedly alert +to the possibilities of white-collar crime and electronic fraud. + +And William J. Cook in particular was a rising star in +electronic crime-busting. He and his fellow federal prosecutors +at the U.S. Attorney's office in Chicago had a tight relation +with the Secret Service, especially go-getting Chicago-based agent +Timothy Foley. While Cook and his Department of Justice colleagues +plotted strategy, Foley was their man on the street. + +Throughout the 1980s, the federal government had given prosecutors +an armory of new, untried legal tools against computer crime. +Cook and his colleagues were pioneers in the use of these new statutes +in the real-life cut-and-thrust of the federal courtroom. + +On October 2, 1986, the US Senate had passed the +"Computer Fraud and Abuse Act" unanimously, but there +were pitifully few convictions under this statute. +Cook's group took their name from this statute, +since they were determined to transform this powerful but +rather theoretical Act of Congress into a real-life engine +of legal destruction against computer fraudsters and scofflaws. + +It was not a question of merely discovering crimes, +investigating them, and then trying and punishing their +perpetrators. The Chicago unit, like most everyone else +in the business, already KNEW who the bad guys were: +the Legion of Doom and the writers and editors of Phrack. +The task at hand was to find some legal means of putting +these characters away. + +This approach might seem a bit dubious, to someone not +acquainted with the gritty realities of prosecutorial work. +But prosecutors don't put people in jail for crimes +they have committed; they put people in jail for crimes +they have committed THAT CAN BE PROVED IN COURT. +Chicago federal police put Al Capone in prison +for income-tax fraud. Chicago is a big town, +with a rough-and-ready bare-knuckle tradition +on both sides of the law. + +Fry Guy had broken the case wide open and alerted telco security +to the scope of the problem. But Fry Guy's crimes would not +put the Atlanta Three behind bars--much less the wacko underground +journalists of Phrack. So on July 22, 1989, the same day that +Fry Guy was raided in Indiana, the Secret Service descended upon +the Atlanta Three. + +This was likely inevitable. By the summer of 1989, law enforcement +were closing in on the Atlanta Three from at least six directions at once. +First, there were the leads from Fry Guy, which had led to the DNR registers +being installed on the lines of the Atlanta Three. The DNR evidence alone +would have finished them off, sooner or later. + +But second, the Atlanta lads were already well-known to Control-C +and his telco security sponsors. LoD's contacts with telco security +had made them overconfident and even more boastful than usual; +they felt that they had powerful friends in high places, +and that they were being openly tolerated by telco security. +But BellSouth's Intrusion Task Force were hot on the trail of LoD +and sparing no effort or expense. + +The Atlanta Three had also been identified by name and listed +on the extensive anti-hacker files maintained, and retailed for pay, +by private security operative John Maxfield of Detroit. +Maxfield, who had extensive ties to telco security +and many informants in the underground, was a bete noire +of the Phrack crowd, and the dislike was mutual. + + +The Atlanta Three themselves had written articles for Phrack. +This boastful act could not possibly escape telco and law enforcement +attention. + +"Knightmare," a high-school age hacker from Arizona, +was a close friend and disciple of Atlanta LoD, +but he had been nabbed by the formidable Arizona +Organized Crime and Racketeering Unit. Knightmare was +on some of LoD's favorite boards--"Black Ice" in particular-- +and was privy to their secrets. And to have Gail Thackeray, +the Assistant Attorney General of Arizona, on one's trail +was a dreadful peril for any hacker. + +And perhaps worst of all, Prophet had committed a major blunder +by passing an illicitly copied BellSouth computer-file to Knight Lightning, +who had published it in Phrack. This, as we will see, was an act of dire +consequence for almost everyone concerned. + +On July 22, 1989, the Secret Service showed up at the Leftist's house, +where he lived with his parents. A massive squad of some twenty officers +surrounded the building: Secret Service, federal marshals, local police, +possibly BellSouth telco security; it was hard to tell in the crush. +Leftist's dad, at work in his basement office, first noticed +a muscular stranger in plain clothes crashing through the +back yard with a drawn pistol. As more strangers poured +into the house, Leftist's dad naturally assumed there was +an armed robbery in progress. + +Like most hacker parents, Leftist's mom and dad had only the vaguest +notions of what their son had been up to all this time. Leftist had +a day-job repairing computer hardware. His obsession with computers +seemed a bit odd, but harmless enough, and likely to produce a well- +paying career. The sudden, overwhelming raid left Leftist's +parents traumatized. + +The Leftist himself had been out after work with his co-workers, +surrounding a couple of pitchers of margaritas. As he came trucking +on tequila-numbed feet up the pavement, toting a bag full of floppy-disks, +he noticed a large number of unmarked cars parked in his driveway. +All the cars sported tiny microwave antennas. + +The Secret Service had knocked the front door off its hinges, +almost flattening his mom. + +Inside, Leftist was greeted by Special Agent James Cool +of the US Secret Service, Atlanta office. Leftist was flabbergasted. +He'd never met a Secret Service agent before. He could not imagine +that he'd ever done anything worthy of federal attention. +He'd always figured that if his activities became intolerable, +one of his contacts in telco security would give him a private +phone-call and tell him to knock it off. + +But now Leftist was pat-searched for weapons by grim professionals, +and his bag of floppies was quickly seized. He and his parents were +all shepherded into separate rooms and grilled at length as a score +of officers scoured their home for anything electronic. + +Leftist was horrified as his treasured IBM AT personal computer +with its forty-meg hard disk, and his recently purchased 80386 IBM-clone +with a whopping hundred-meg hard disk, both went swiftly out the door +in Secret Service custody. They also seized all his disks, all his notebooks, +and a tremendous booty in dogeared telco documents that Leftist had snitched +out of trash dumpsters. + +Leftist figured the whole thing for a big misunderstanding. +He'd never been into MILITARY computers. He wasn't a SPY or a COMMUNIST. +He was just a good ol' Georgia hacker, and now he just wanted all these +people out of the house. But it seemed they wouldn't go until he made +some kind of statement. + +And so, he levelled with them. + +And that, Leftist said later from his federal prison camp in Talladega, +Alabama, was a big mistake. The Atlanta area was unique, +in that it had three members of the Legion of Doom who actually +occupied more or less the same physical locality. Unlike the rest +of LoD, who tended to associate by phone and computer, +Atlanta LoD actually WERE "tightly knit." It was no real +surprise that the Secret Service agents apprehending Urvile +at the computer-labs at Georgia Tech, would discover Prophet +with him as well. + +Urvile, a 21-year-old Georgia Tech student in polymer chemistry, +posed quite a puzzling case for law enforcement. Urvile--also known +as "Necron 99," as well as other handles, for he tended to change his +cover-alias about once a month--was both an accomplished hacker +and a fanatic simulation-gamer. + +Simulation games are an unusual hobby; but then hackers are unusual people, +and their favorite pastimes tend to be somewhat out of the ordinary. +The best-known American simulation game is probably "Dungeons & Dragons," +a multi-player parlor entertainment played with paper, maps, pencils, +statistical tables and a variety of oddly-shaped dice. Players pretend +to be heroic characters exploring a wholly-invented fantasy world. +The fantasy worlds of simulation gaming are commonly pseudo-medieval, +involving swords and sorcery--spell-casting wizards, knights in armor, +unicorns and dragons, demons and goblins. + +Urvile and his fellow gamers preferred their fantasies highly technological. +They made use of a game known as "G.U.R.P.S.," the "Generic Universal Role +Playing System," published by a company called Steve Jackson Games (SJG). + +"G.U.R.P.S." served as a framework for creating a wide variety of artificial +fantasy worlds. Steve Jackson Games published a smorgasboard of books, +full of detailed information and gaming hints, which were used to flesh-out +many different fantastic backgrounds for the basic GURPS framework. +Urvile made extensive use of two SJG books called GURPS High-Tech +and GURPS Special Ops. + +In the artificial fantasy-world of GURPS Special Ops, +players entered a modern fantasy of intrigue and international espionage. +On beginning the game, players started small and powerless, +perhaps as minor-league CIA agents or penny-ante arms dealers. +But as players persisted through a series of game sessions +(game sessions generally lasted for hours, over long, +elaborate campaigns that might be pursued for months on end) +then they would achieve new skills, new knowledge, new power. +They would acquire and hone new abilities, such as marksmanship, +karate, wiretapping, or Watergate burglary. They could also win +various kinds of imaginary booty, like Berettas, or martini shakers, +or fast cars with ejection seats and machine-guns under the headlights. + +As might be imagined from the complexity of these games, +Urvile's gaming notes were very detailed and extensive. +Urvile was a "dungeon-master," inventing scenarios +for his fellow gamers, giant simulated adventure-puzzles +for his friends to unravel. Urvile's game notes covered +dozens of pages with all sorts of exotic lunacy, all about +ninja raids on Libya and break-ins on encrypted Red Chinese supercomputers. +His notes were written on scrap-paper and kept in loose-leaf binders. + +The handiest scrap paper around Urvile's college digs were the many pounds of +BellSouth printouts and documents that he had snitched out of telco dumpsters. +His notes were written on the back of misappropriated telco property. +Worse yet, the gaming notes were chaotically interspersed with Urvile's +hand-scrawled records involving ACTUAL COMPUTER INTRUSIONS that he +had committed. + +Not only was it next to impossible to tell Urvile's fantasy game-notes +from cyberspace "reality," but Urvile himself barely made this distinction. +It's no exaggeration to say that to Urvile it was ALL a game. Urvile was +very bright, highly imaginative, and quite careless of other people's notions +of propriety. His connection to "reality" was not something to which he paid +a great deal of attention. + +Hacking was a game for Urvile. It was an amusement he was carrying out, +it was something he was doing for fun. And Urvile was an obsessive young man. +He could no more stop hacking than he could stop in the middle of +a jigsaw puzzle, or stop in the middle of reading a Stephen Donaldson +fantasy trilogy. (The name "Urvile" came from a best-selling Donaldson novel.) + +Urvile's airy, bulletproof attitude seriously annoyed his interrogators. +First of all, he didn't consider that he'd done anything wrong. +There was scarcely a shred of honest remorse in him. On the contrary, +he seemed privately convinced that his police interrogators were operating +in a demented fantasy-world all their own. Urvile was too polite +and well-behaved to say this straight-out, but his reactions were askew +and disquieting. + +For instance, there was the business about LoD's ability +to monitor phone-calls to the police and Secret Service. +Urvile agreed that this was quite possible, and posed +no big problem for LoD. In fact, he and his friends +had kicked the idea around on the "Black Ice" board, +much as they had discussed many other nifty notions, +such as building personal flame-throwers and jury-rigging +fistfulls of blasting-caps. They had hundreds of dial-up numbers +for government agencies that they'd gotten through scanning Atlanta phones, +or had pulled from raided VAX/VMS mainframe computers. + +Basically, they'd never gotten around to listening in on the cops +because the idea wasn't interesting enough to bother with. +Besides, if they'd been monitoring Secret Service phone calls, +obviously they'd never have been caught in the first place. Right? + +The Secret Service was less than satisfied with this rapier-like hacker logic. + +Then there was the issue of crashing the phone system. No problem, +Urvile admitted sunnily. Atlanta LoD could have shut down phone service +all over Atlanta any time they liked. EVEN THE 911 SERVICE? +Nothing special about that, Urvile explained patiently. +Bring the switch to its knees, with say the UNIX "makedir" bug, +and 911 goes down too as a matter of course. The 911 system +wasn't very interesting, frankly. It might be tremendously +interesting to cops (for odd reasons of their own), but as +technical challenges went, the 911 service was yawnsville. + +So of course the Atlanta Three could crash service. +They probably could have crashed service all over +BellSouth territory, if they'd worked at it for a while. +But Atlanta LoD weren't crashers. Only losers and rodents +were crashers. LoD were ELITE. + +Urvile was privately convinced that sheer technical +expertise could win him free of any kind of problem. +As far as he was concerned, elite status in the digital +underground had placed him permanently beyond the intellectual +grasp of cops and straights. Urvile had a lot to learn. + +Of the three LoD stalwarts, Prophet was in the most direct trouble. +Prophet was a UNIX programming expert who burrowed in and out +of the Internet as a matter of course. He'd started his hacking +career at around age 14, meddling with a UNIX mainframe system +at the University of North Carolina. + +Prophet himself had written the handy Legion of Doom +file "UNIX Use and Security From the Ground Up." +UNIX (pronounced "you-nicks") is a powerful, +flexible computer operating-system, for multi-user, +multi-tasking computers. In 1969, when UNIX was created +in Bell Labs, such computers were exclusive to large +corporations and universities, but today UNIX is run +on thousands of powerful home machines. UNIX was +particularly well-suited to telecommunications programming, +and had become a standard in the field. Naturally, UNIX +also became a standard for the elite hacker and phone phreak. +Lately, Prophet had not been so active as Leftist and Urvile, +but Prophet was a recidivist. In 1986, when he was eighteen, +Prophet had been convicted of "unauthorized access +to a computer network" in North Carolina. He'd been +discovered breaking into the Southern Bell Data Network, +a UNIX-based internal telco network supposedly closed to the public. +He'd gotten a typical hacker sentence: six months suspended, +120 hours community service, and three years' probation. + +After that humiliating bust, Prophet had gotten rid of most of his +tonnage of illicit phreak and hacker data, and had tried to go straight. +He was, after all, still on probation. But by the autumn of 1988, +the temptations of cyberspace had proved too much for young Prophet, +and he was shoulder-to-shoulder with Urvile and Leftist into some +of the hairiest systems around. + +In early September 1988, he'd broken into BellSouth's centralized +automation system, AIMSX or "Advanced Information Management System." +AIMSX was an internal business network for BellSouth, where telco +employees stored electronic mail, databases, memos, and calendars, +and did text processing. Since AIMSX did not have public dial-ups, +it was considered utterly invisible to the public, and was not well-secured +--it didn't even require passwords. Prophet abused an account known +as "waa1," the personal account of an unsuspecting telco employee. +Disguised as the owner of waa1, Prophet made about ten visits to AIMSX. + +Prophet did not damage or delete anything in the system. +His presence in AIMSX was harmless and almost invisible. +But he could not rest content with that. + +One particular piece of processed text on AIMSX was a telco document +known as "Bell South Standard Practice 660-225-104SV Control Office +Administration of Enhanced 911 Services for Special Services +and Major Account Centers dated March 1988." + +Prophet had not been looking for this document. It was merely one +among hundreds of similar documents with impenetrable titles. +However, having blundered over it in the course of his illicit +wanderings through AIMSX, he decided to take it with him as a trophy. +It might prove very useful in some future boasting, bragging, +and strutting session. So, some time in September 1988, +Prophet ordered the AIMSX mainframe computer to copy this document +(henceforth called simply called "the E911 Document") and to transfer +this copy to his home computer. + +No one noticed that Prophet had done this. He had "stolen" +the E911 Document in some sense, but notions of property +in cyberspace can be tricky. BellSouth noticed nothing wrong, +because BellSouth still had their original copy. They had not +been "robbed" of the document itself. Many people were supposed +to copy this document--specifically, people who worked for the +nineteen BellSouth "special services and major account centers," +scattered throughout the Southeastern United States. That was +what it was for, why it was present on a computer network +in the first place: so that it could be copied and read-- +by telco employees. But now the data had been copied +by someone who wasn't supposed to look at it. + +Prophet now had his trophy. But he further decided to store +yet another copy of the E911 Document on another person's computer. +This unwitting person was a computer enthusiast named Richard Andrews +who lived near Joliet, Illinois. Richard Andrews was a UNIX programmer +by trade, and ran a powerful UNIX board called "Jolnet," in the basement +of his house. + +Prophet, using the handle "Robert Johnson," had obtained an account +on Richard Andrews' computer. And there he stashed the E911 Document, +by storing it in his own private section of Andrews' computer. + +Why did Prophet do this? If Prophet had eliminated the E911 Document +from his own computer, and kept it hundreds of miles away, on another machine, under an +alias, then he might have been fairly safe from discovery and prosecution-- +although his sneaky action had certainly put the unsuspecting Richard Andrews +at risk. + +But, like most hackers, Prophet was a pack-rat for illicit data. +When it came to the crunch, he could not bear to part from his trophy. +When Prophet's place in Decatur, Georgia was raided in July 1989, +there was the E911 Document, a smoking gun. And there was Prophet +in the hands of the Secret Service, doing his best to "explain." + +Our story now takes us away from the Atlanta Three and their raids +of the Summer of 1989. We must leave Atlanta Three "cooperating fully" +with their numerous investigators. And all three of them did cooperate, +as their Sentencing Memorandum from the US District Court of the +Northern Division of Georgia explained--just before all three of them +were sentenced to various federal prisons in November 1990. + +We must now catch up on the other aspects of the war on the Legion of Doom. +The war on the Legion was a war on a network--in fact, a network of three +networks, which intertwined and interrelated in a complex fashion. +The Legion itself, with Atlanta LoD, and their hanger-on Fry Guy, +were the first network. The second network was Phrack magazine, +with its editors and contributors. + +The third network involved the electronic circle around a hacker +known as "Terminus." + +The war against these hacker networks was carried out by +a law enforcement network. Atlanta LoD and Fry Guy +were pursued by USSS agents and federal prosecutors in Atlanta, +Indiana, and Chicago. "Terminus" found himself pursued by USSS +and federal prosecutors from Baltimore and Chicago. And the war +against Phrack was almost entirely a Chicago operation. + +The investigation of Terminus involved a great deal of energy, +mostly from the Chicago Task Force, but it was to be the least-known +and least-publicized of the Crackdown operations. Terminus, who lived +in Maryland, was a UNIX programmer and consultant, fairly well-known +(under his given name) in the UNIX community, as an acknowledged expert +on AT&T minicomputers. Terminus idolized AT&T, especially Bellcore, +and longed for public recognition as a UNIX expert; his highest ambition +was to work for Bell Labs. + +But Terminus had odd friends and a spotted history. +Terminus had once been the subject of an admiring interview +in Phrack (Volume II, Issue 14, Phile 2--dated May 1987). +In this article, Phrack co-editor Taran King described +"Terminus" as an electronics engineer, 5'9", brown-haired, +born in 1959--at 28 years old, quite mature for a hacker. + +Terminus had once been sysop of a phreak/hack underground board +called "MetroNet," which ran on an Apple II. Later he'd replaced +"MetroNet" with an underground board called "MegaNet," +specializing in IBMs. In his younger days, Terminus had written +one of the very first and most elegant code-scanning programs +for the IBM-PC. This program had been widely distributed +in the underground. Uncounted legions of PC-owning phreaks and +hackers had used Terminus's scanner program to rip-off telco codes. +This feat had not escaped the attention of telco security; +it hardly could, since Terminus's earlier handle, "Terminal Technician," +was proudly written right on the program. + +When he became a full-time computer professional +(specializing in telecommunications programming), +he adopted the handle Terminus, meant to indicate that he +had "reached the final point of being a proficient hacker." +He'd moved up to the UNIX-based "Netsys" board on an AT&T computer, +with four phone lines and an impressive 240 megs of storage. +"Netsys" carried complete issues of Phrack, and Terminus was +quite friendly with its publishers, Taran King and Knight Lightning. + +In the early 1980s, Terminus had been a regular on Plovernet, +Pirate-80, Sherwood Forest and Shadowland, all well-known pirate boards, +all heavily frequented by the Legion of Doom. As it happened, Terminus +was never officially "in LoD," because he'd never been given the official +LoD high-sign and back-slap by Legion maven Lex Luthor. Terminus had +never physically met anyone from LoD. But that scarcely mattered much-- +the Atlanta Three themselves had never been officially vetted by Lex, either. + +As far as law enforcement was concerned, the issues were clear. +Terminus was a full-time, adult computer professional +with particular skills at AT&T software and hardware-- +but Terminus reeked of the Legion of Doom and the underground. + +On February 1, 1990--half a month after the Martin Luther King Day Crash-- +USSS agents Tim Foley from Chicago, and Jack Lewis from the Baltimore office, +accompanied by AT&T security officer Jerry Dalton, travelled to Middle Town, +Maryland. There they grilled Terminus in his home (to the stark terror of +his wife and small children), and, in their customary fashion, hauled his +computers out the door. + +The Netsys machine proved to contain a plethora of arcane UNIX software-- +proprietary source code formally owned by AT&T. Software such as: +UNIX System Five Release 3.2; UNIX SV Release 3.1; UUCP communications +software; KORN SHELL; RFS; IWB; WWB; DWB; the C++ programming language; +PMON; TOOL CHEST; QUEST; DACT, and S FIND. + +In the long-established piratical tradition of the underground, +Terminus had been trading this illicitly-copied software with +a small circle of fellow UNIX programmers. Very unwisely, +he had stored seven years of his electronic mail on his Netsys machine, +which documented all the friendly arrangements he had made with +his various colleagues. + +Terminus had not crashed the AT&T phone system on January 15. +He was, however, blithely running a not-for-profit AT&T +software-piracy ring. This was not an activity AT&T found amusing. +AT&T security officer Jerry Dalton valued this "stolen" property +at over three hundred thousand dollars. + +AT&T's entry into the tussle of free enterprise had been complicated +by the new, vague groundrules of the information economy. +Until the break-up of Ma Bell, AT&T was forbidden to sell +computer hardware or software. Ma Bell was the phone company; +Ma Bell was not allowed to use the enormous revenue from +telephone utilities, in order to finance any entry into +the computer market. + +AT&T nevertheless invented the UNIX operating system. +And somehow AT&T managed to make UNIX a minor source of income. +Weirdly, UNIX was not sold as computer software, +but actually retailed under an obscure regulatory +exemption allowing sales of surplus equipment and scrap. +Any bolder attempt to promote or retail UNIX would have +aroused angry legal opposition from computer companies. +Instead, UNIX was licensed to universities, at modest rates, +where the acids of academic freedom ate away steadily at AT&T's +proprietary rights. + +Come the breakup, AT&T recognized that UNIX was a potential gold-mine. +By now, large chunks of UNIX code had been created that were not AT&T's, +and were being sold by others. An entire rival UNIX-based operating system +had arisen in Berkeley, California (one of the world's great founts of +ideological hackerdom). Today, "hackers" commonly consider "Berkeley UNIX" +to be technically superior to AT&T's "System V UNIX," but AT&T has not +allowed mere technical elegance to intrude on the real-world business +of marketing proprietary software. AT&T has made its own code deliberately +incompatible with other folks' UNIX, and has written code that it can prove +is copyrightable, even if that code happens to be somewhat awkward--"kludgey." +AT&T UNIX user licenses are serious business agreements, replete with very +clear copyright statements and non-disclosure clauses. + +AT&T has not exactly kept the UNIX cat in the bag, +but it kept a grip on its scruff with some success. +By the rampant, explosive standards of software piracy, +AT&T UNIX source code is heavily copyrighted, well-guarded, +well-licensed. UNIX was traditionally run only on +mainframe machines, owned by large groups of suit-and-tie +professionals, rather than on bedroom machines where +people can get up to easy mischief. + +And AT&T UNIX source code is serious high-level programming. +The number of skilled UNIX programmers with any actual motive +to swipe UNIX source code is small. It's tiny, compared to +the tens of thousands prepared to rip-off, say, entertaining +PC games like "Leisure Suit Larry." + +But by 1989, the warez-d00d underground, in the persons of Terminus +and his friends, was gnawing at AT&T UNIX. And the property in question +was not sold for twenty bucks over the counter at the local branch of +Babbage's or Egghead's; this was massive, sophisticated, multi-line, +multi-author corporate code worth tens of thousands of dollars. + +It must be recognized at this point that Terminus's purported ring of UNIX +software pirates had not actually made any money from their suspected crimes. +The $300,000 dollar figure bandied about for the contents of Terminus's +computer did not mean that Terminus was in actual illicit possession +of three hundred thousand of AT&T's dollars. Terminus was shipping +software back and forth, privately, person to person, for free. +He was not making a commercial business of piracy. He hadn't +asked for money; he didn't take money. He lived quite modestly. + +AT&T employees--as well as freelance UNIX consultants, like Terminus-- +commonly worked with "proprietary" AT&T software, both in the office +and at home on their private machines. AT&T rarely sent security officers +out to comb the hard disks of its consultants. Cheap freelance UNIX +contractors were quite useful to AT&T; they didn't have health insurance +or retirement programs, much less union membership in the Communication +Workers of America. They were humble digital drudges, wandering with mop +and bucket through the Great Technological Temple of AT&T; but when the +Secret Service arrived at their homes, it seemed they were eating with +company silverware and sleeping on company sheets! Outrageously, they +behaved as if the things they worked with every day belonged to them! + +And these were no mere hacker teenagers with their hands full +of trash-paper and their noses pressed to the corporate windowpane. +These guys were UNIX wizards, not only carrying AT&T data in their +machines and their heads, but eagerly networking about it, +over machines that were far more powerful than anything previously +imagined in private hands. How do you keep people disposable, +yet assure their awestruck respect for your property? It was a dilemma. + +Much UNIX code was public-domain, available for free. Much "proprietary" +UNIX code had been extensively re-written, perhaps altered so much that it +became an entirely new product--or perhaps not. Intellectual property rights +for software developers were, and are, extraordinarily complex and confused. +And software "piracy," like the private copying of videos, is one of the most +widely practiced "crimes" in the world today. + +The USSS were not experts in UNIX or familiar with the customs of its use. +The United States Secret Service, considered as a body, did not have one single +person in it who could program in a UNIX environment--no, not even one. +The Secret Service WERE making extensive use of expert help, but the "experts" +they had chosen were AT&T and Bellcore security officials, the very victims of +the purported crimes under investigation, the very people whose interest in +AT&T's "proprietary" software was most pronounced. + +On February 6, 1990, Terminus was arrested by Agent Lewis. +Eventually, Terminus would be sent to prison for his illicit +use of a piece of AT&T software. + +The issue of pirated AT&T software would bubble along in the background +during the war on the Legion of Doom. Some half-dozen of Terminus's on-line +acquaintances, including people in Illinois, Texas and California, +were grilled by the Secret Service in connection with the illicit +copying of software. Except for Terminus, however, none were charged +with a crime. None of them shared his peculiar prominence in the +hacker underground. + +But that did not mean that these people would, or could, +stay out of trouble. The transferral of illicit data in +cyberspace is hazy and ill-defined business, with paradoxical +dangers for everyone concerned: hackers, signal carriers, +board owners, cops, prosecutors, even random passers-by. +Sometimes, well-meant attempts to avert trouble +or punish wrongdoing bring more trouble than +would simple ignorance, indifference or impropriety. + +Terminus's "Netsys" board was not a common-or-garden +bulletin board system, though it had most of the usual +functions of a board. Netsys was not a stand-alone machine, +but part of the globe-spanning "UUCP" cooperative network. +The UUCP network uses a set of Unix software programs called +"Unix-to-Unix Copy," which allows Unix systems to throw data to +one another at high speed through the public telephone network. +UUCP is a radically decentralized, not-for-profit network of UNIX computers. +There are tens of thousands of these UNIX machines. Some are small, +but many are powerful and also link to other networks. UUCP has +certain arcane links to major networks such as JANET, EasyNet, BITNET, +JUNET, VNET, DASnet, PeaceNet and FidoNet, as well as the gigantic Internet. +(The so-called "Internet" is not actually a network itself, but rather an +"internetwork" connections standard that allows several globe-spanning +computer networks to communicate with one another. Readers fascinated +by the weird and intricate tangles of modern computer networks may enjoy +John S. Quarterman's authoritative 719-page explication, The Matrix, +Digital Press, 1990.) + +A skilled user of Terminus' UNIX machine could send and receive +electronic mail from almost any major computer network in the world. +Netsys was not called a "board" per se, but rather a "node." +"Nodes" were larger, faster, and more sophisticated than mere "boards," +and for hackers, to hang out on internationally-connected "nodes" +was quite the step up from merely hanging out on local "boards." + +Terminus's Netsys node in Maryland had a number of direct +links to other, similar UUCP nodes, run by people who shared his +interests and at least something of his free-wheeling attitude. +One of these nodes was Jolnet, owned by Richard Andrews, who, +like Terminus, was an independent UNIX consultant. +Jolnet also ran UNIX, and could be contacted at high speed +by mainframe machines from all over the world. Jolnet was +quite a sophisticated piece of work, technically speaking, +but it was still run by an individual, as a private, +not-for-profit hobby. Jolnet was mostly used by other +UNIX programmers--for mail, storage, and access to networks. +Jolnet supplied access network access to about two hundred people, +as well as a local junior college. + +Among its various features and services, Jolnet also carried +Phrack magazine. + +For reasons of his own, Richard Andrews had become suspicious +of a new user called "Robert Johnson." Richard Andrews +took it upon himself to have a look at what "Robert Johnson" +was storing in Jolnet. And Andrews found the E911 Document. + +"Robert Johnson" was the Prophet from the Legion of Doom, +and the E911 Document was illicitly copied data from Prophet's +raid on the BellSouth computers. + +The E911 Document, a particularly illicit piece of digital property, +was about to resume its long, complex, and disastrous career. + +It struck Andrews as fishy that someone not a telephone employee +should have a document referring to the "Enhanced 911 System." +Besides, the document itself bore an obvious warning. + +"WARNING: NOT FOR USE OR DISCLOSURE OUTSIDE BELLSOUTH +OR ANY OF ITS SUBSIDIARIES EXCEPT UNDER WRITTEN AGREEMENT." + +These standard nondisclosure tags are often appended to all sorts +of corporate material. Telcos as a species are particularly notorious +for stamping most everything in sight as "not for use or disclosure." +Still, this particular piece of data was about the 911 System. +That sounded bad to Rich Andrews. + +Andrews was not prepared to ignore this sort of trouble. +He thought it would be wise to pass the document along +to a friend and acquaintance on the UNIX network, for consultation. +So, around September 1988, Andrews sent yet another copy of the +E911 Document electronically to an AT&T employee, one Charles Boykin, +who ran a UNIX-based node called "attctc" in Dallas, Texas. + +"Attctc" was the property of AT&T, and was run from AT&T's +Customer Technology Center in Dallas, hence the name "attctc." +"Attctc" was better-known as "Killer," the name of the machine +that the system was running on. "Killer" was a hefty, powerful, +AT&T 3B2 500 model, a multi-user, multi-tasking UNIX platform +with 32 meg of memory and a mind-boggling 3.2 Gigabytes of storage. +When Killer had first arrived in Texas, in 1985, the 3B2 had been +one of AT&T's great white hopes for going head-to-head with IBM +for the corporate computer-hardware market. "Killer" had been shipped +to the Customer Technology Center in the Dallas Infomart, essentially +a high-technology mall, and there it sat, a demonstration model. + +Charles Boykin, a veteran AT&T hardware and digital communications expert, +was a local technical backup man for the AT&T 3B2 system. As a display model +in the Infomart mall, "Killer" had little to do, and it seemed a shame +to waste the system's capacity. So Boykin ingeniously wrote some UNIX +bulletin-board software for "Killer," and plugged the machine in to the +local phone network. "Killer's" debut in late 1985 made it the first +publicly available UNIX site in the state of Texas. Anyone who wanted to +play was welcome. + +The machine immediately attracted an electronic community. +It joined the UUCP network, and offered network links +to over eighty other computer sites, all of which became dependent +on Killer for their links to the greater world of cyberspace. +And it wasn't just for the big guys; personal computer users +also stored freeware programs for the Amiga, the Apple, +the IBM and the Macintosh on Killer's vast 3,200 meg archives. +At one time, Killer had the largest library of public-domain +Macintosh software in Texas. + +Eventually, Killer attracted about 1,500 users, +all busily communicating, uploading and downloading, +getting mail, gossipping, and linking to arcane +and distant networks. + +Boykin received no pay for running Killer. He considered +it good publicity for the AT&T 3B2 system (whose sales were +somewhat less than stellar), but he also simply enjoyed +the vibrant community his skill had created. He gave away +the bulletin-board UNIX software he had written, free of charge. + +In the UNIX programming community, Charlie Boykin had the +reputation of a warm, open-hearted, level-headed kind of guy. +In 1989, a group of Texan UNIX professionals voted Boykin +"System Administrator of the Year." He was considered +a fellow you could trust for good advice. + +In September 1988, without warning, the E911 Document +came plunging into Boykin's life, forwarded by Richard Andrews. +Boykin immediately recognized that the Document was hot property. +He was not a voice-communications man, and knew little about +the ins and outs of the Baby Bells, but he certainly knew what +the 911 System was, and he was angry to see confidential data +about it in the hands of a nogoodnik. This was clearly a +matter for telco security. So, on September 21, 1988, Boykin +made yet ANOTHER copy of the E911 Document and passed this +one along to a professional acquaintance of his, one Jerome Dalton, +from AT&T Corporate Information Security. Jerry Dalton was the +very fellow who would later raid Terminus's house. + +From AT&T's security division, the E911 Document went to Bellcore. + +Bellcore (or BELL COmmunications REsearch) had once been the central +laboratory of the Bell System. Bell Labs employees had invented +the UNIX operating system. Now Bellcore was a quasi-independent, +jointly owned company that acted as the research arm for all seven +of the Baby Bell RBOCs. Bellcore was in a good position to co-ordinate +security technology and consultation for the RBOCs, and the gentleman in +charge of this effort was Henry M. Kluepfel, a veteran of the Bell System +who had worked there for twenty-four years. + +On October 13, 1988, Dalton passed the E911 Document to Henry Kluepfel. +Kluepfel, a veteran expert witness in telecommunications fraud and +computer-fraud cases, had certainly seen worse trouble than this. +He recognized the document for what it was: a trophy from a hacker break-in. + +However, whatever harm had been done in the intrusion was presumably old news. +At this point there seemed little to be done. Kluepfel made a careful note +of the circumstances and shelved the problem for the time being. + +Whole months passed. + +February 1989 arrived. The Atlanta Three were living it up +in Bell South's switches, and had not yet met their comeuppance. +The Legion was thriving. So was Phrack magazine. +A good six months had passed since Prophet's AIMSX break-in. +Prophet, as hackers will, grew weary of sitting on his laurels. +"Knight Lightning" and "Taran King," the editors of Phrack, +were always begging Prophet for material they could publish. +Prophet decided that the heat must be off by this time, +and that he could safely brag, boast, and strut. + +So he sent a copy of the E911 Document--yet another one-- +from Rich Andrews' Jolnet machine to Knight Lightning's +BITnet account at the University of Missouri. +Let's review the fate of the document so far. + +0. The original E911 Document. This in the AIMSX system +on a mainframe computer in Atlanta, available to hundreds of people, +but all of them, presumably, BellSouth employees. An unknown number +of them may have their own copies of this document, but they are all +professionals and all trusted by the phone company. + +1. Prophet's illicit copy, at home on his own computer in Decatur, Georgia. + +2. Prophet's back-up copy, stored on Rich Andrew's Jolnet machine + in the basement of Rich Andrews' house near Joliet Illinois. + +3. Charles Boykin's copy on "Killer" in Dallas, Texas, + sent by Rich Andrews from Joliet. + +4. Jerry Dalton's copy at AT&T Corporate Information Security in New Jersey, + sent from Charles Boykin in Dallas. + +5. Henry Kluepfel's copy at Bellcore security headquarters in New Jersey, + sent by Dalton. +6. Knight Lightning's copy, sent by Prophet from Rich Andrews' machine, + and now in Columbia, Missouri. + +We can see that the "security" situation of this proprietary document, +once dug out of AIMSX, swiftly became bizarre. Without any money +changing hands, without any particular special effort, this data +had been reproduced at least six times and had spread itself all over +the continent. By far the worst, however, was yet to come. + +In February 1989, Prophet and Knight Lightning bargained electronically +over the fate of this trophy. Prophet wanted to boast, but, at the same time, +scarcely wanted to be caught. + +For his part, Knight Lightning was eager to publish as much of the document +as he could manage. Knight Lightning was a fledgling political-science major +with a particular interest in freedom-of-information issues. He would gladly +publish most anything that would reflect glory on the prowess of the +underground and embarrass the telcos. However, Knight Lightning himself +had contacts in telco security, and sometimes consulted them on material +he'd received that might be too dicey for publication. + +Prophet and Knight Lightning decided to edit the E911 Document +so as to delete most of its identifying traits. First of all, +its large "NOT FOR USE OR DISCLOSURE" warning had to go. +Then there were other matters. For instance, it listed +the office telephone numbers of several BellSouth 911 +specialists in Florida. If these phone numbers were +published in Phrack, the BellSouth employees involved +would very likely be hassled by phone phreaks, +which would anger BellSouth no end, and pose a +definite operational hazard for both Prophet and Phrack. + +So Knight Lightning cut the Document almost in half, +removing the phone numbers and some of the touchier +and more specific information. He passed it back +electronically to Prophet; Prophet was still nervous, +so Knight Lightning cut a bit more. They finally agreed +that it was ready to go, and that it would be published +in Phrack under the pseudonym, "The Eavesdropper." + +And this was done on February 25, 1989. + +The twenty-fourth issue of Phrack featured a chatty interview +with co-ed phone-phreak "Chanda Leir," three articles on BITNET +and its links to other computer networks, an article on 800 and 900 +numbers by "Unknown User," "VaxCat's" article on telco basics +(slyly entitled "Lifting Ma Bell's Veil of Secrecy,)" and +the usual "Phrack World News." + +The News section, with painful irony, featured an extended account +of the sentencing of "Shadowhawk," an eighteen-year-old Chicago hacker +who had just been put in federal prison by William J. Cook himself. + +And then there were the two articles by "The Eavesdropper." +The first was the edited E911 Document, now titled +"Control Office Administration Of Enhanced 911 Services +for Special Services and Major Account Centers." +Eavesdropper's second article was a glossary of terms +explaining the blizzard of telco acronyms and buzzwords +in the E911 Document. + +The hapless document was now distributed, in the usual Phrack routine, +to a good one hundred and fifty sites. Not a hundred and fifty PEOPLE, +mind you--a hundred and fifty SITES, some of these sites linked to UNIX +nodes or bulletin board systems, which themselves had readerships of tens, +dozens, even hundreds of people. + +This was February 1989. Nothing happened immediately. +Summer came, and the Atlanta crew were raided by the Secret Service. +Fry Guy was apprehended. Still nothing whatever happened to Phrack. +Six more issues of Phrack came out, 30 in all, more or less on +a monthly schedule. Knight Lightning and co-editor Taran King +went untouched. + +Phrack tended to duck and cover whenever the heat came down. +During the summer busts of 1987--(hacker busts tended to cluster in summer, +perhaps because hackers were easier to find at home than in college)-- +Phrack had ceased publication for several months, and laid low. +Several LoD hangers-on had been arrested, but nothing had happened +to the Phrack crew, the premiere gossips of the underground. +In 1988, Phrack had been taken over by a new editor, +"Crimson Death," a raucous youngster with a taste for anarchy files. +1989, however, looked like a bounty year for the underground. +Knight Lightning and his co-editor Taran King took up the reins again, +and Phrack flourished throughout 1989. Atlanta LoD went down hard in +the summer of 1989, but Phrack rolled merrily on. Prophet's E911 Document +seemed unlikely to cause Phrack any trouble. By January 1990, +it had been available in Phrack for almost a year. Kluepfel and Dalton, +officers of Bellcore and AT&T security, had possessed the document +for sixteen months--in fact, they'd had it even before Knight Lightning +himself, and had done nothing in particular to stop its distribution. +They hadn't even told Rich Andrews or Charles Boykin to erase the copies +from their UNIX nodes, Jolnet and Killer. + +But then came the monster Martin Luther King Day Crash of January 15, 1990. + +A flat three days later, on January 18, four agents showed up +at Knight Lightning's fraternity house. One was Timothy Foley, +the second Barbara Golden, both of them Secret Service agents +from the Chicago office. Also along was a University of Missouri +security officer, and Reed Newlin, a security man from Southwestern Bell, +the RBOC having jurisdiction over Missouri. + +Foley accused Knight Lightning of causing the nationwide crash +of the phone system. + +Knight Lightning was aghast at this allegation. On the face of it, +the suspicion was not entirely implausible--though Knight Lightning +knew that he himself hadn't done it. Plenty of hot-dog hackers +had bragged that they could crash the phone system, however. +"Shadowhawk," for instance, the Chicago hacker whom William Cook +had recently put in jail, had several times boasted on boards +that he could "shut down AT&T's public switched network." + +And now this event, or something that looked just like it, +had actually taken place. The Crash had lit a fire under +the Chicago Task Force. And the former fence-sitters at +Bellcore and AT&T were now ready to roll. The consensus +among telco security--already horrified by the skill of +the BellSouth intruders --was that the digital underground +was out of hand. LoD and Phrack must go. And in publishing +Prophet's E911 Document, Phrack had provided law enforcement +with what appeared to be a powerful legal weapon. + +Foley confronted Knight Lightning about the E911 Document. + +Knight Lightning was cowed. He immediately began "cooperating fully" +in the usual tradition of the digital underground. + +He gave Foley a complete run of Phrack, printed out in a set +of three-ring binders. He handed over his electronic mailing list +of Phrack subscribers. Knight Lightning was grilled for four hours +by Foley and his cohorts. Knight Lightning admitted that Prophet +had passed him the E911 Document, and he admitted that he had known +it was stolen booty from a hacker raid on a telephone company. +Knight Lightning signed a statement to this effect, and agreed, +in writing, to cooperate with investigators. + +Next day--January 19, 1990, a Friday --the Secret Service returned +with a search warrant, and thoroughly searched Knight Lightning's +upstairs room in the fraternity house. They took all his floppy disks, +though, interestingly, they left Knight Lightning in possession +of both his computer and his modem. (The computer had no hard disk, +and in Foley's judgement was not a store of evidence.) But this was a +very minor bright spot among Knight Lightning's rapidly multiplying troubles. +By this time, Knight Lightning was in plenty of hot water, not only with +federal police, prosecutors, telco investigators, and university security, +but with the elders of his own campus fraternity, who were outraged +to think that they had been unwittingly harboring a federal computer-criminal. + +On Monday, Knight Lightning was summoned to Chicago, where he was +further grilled by Foley and USSS veteran agent Barbara Golden, this time +with an attorney present. And on Tuesday, he was formally indicted +by a federal grand jury. + +The trial of Knight Lightning, which occurred on July 24-27, 1990, +was the crucial show-trial of the Hacker Crackdown. We will examine +the trial at some length in Part Four of this book. + +In the meantime, we must continue our dogged pursuit of the E911 Document. + +It must have been clear by January 1990 that the E911 Document, +in the form Phrack had published it back in February 1989, +had gone off at the speed of light in at least a hundred +and fifty different directions. To attempt to put this +electronic genie back in the bottle was flatly impossible. + +And yet, the E911 Document was STILL stolen property, +formally and legally speaking. Any electronic transference +of this document, by anyone unauthorized to have it, +could be interpreted as an act of wire fraud. Interstate +transfer of stolen property, including electronic property, +was a federal crime. + +The Chicago Computer Fraud and Abuse Task Force had been assured +that the E911 Document was worth a hefty sum of money. In fact, +they had a precise estimate of its worth from BellSouth security personnel: +$79,449. A sum of this scale seemed to warrant vigorous prosecution. +Even if the damage could not be undone, at least this large sum +offered a good legal pretext for stern punishment of the thieves. +It seemed likely to impress judges and juries. And it could be used +in court to mop up the Legion of Doom. + +The Atlanta crowd was already in the bag, by the time +the Chicago Task Force had gotten around to Phrack. +But the Legion was a hydra-headed thing. In late 89, +a brand-new Legion of Doom board, "Phoenix Project," +had gone up in Austin, Texas. Phoenix Project was sysoped +by no less a man than the Mentor himself, ably assisted by +University of Texas student and hardened Doomster "Erik Bloodaxe." + +As we have seen from his Phrack manifesto, the Mentor was a hacker +zealot who regarded computer intrusion as something close to a moral duty. +Phoenix Project was an ambitious effort, intended to revive the digital +underground to what Mentor considered the full flower of the early 80s. +The Phoenix board would also boldly bring elite hackers face-to-face +with the telco "opposition." On "Phoenix," America's cleverest hackers +would supposedly shame the telco squareheads out of their stick-in-the-mud +attitudes, and perhaps convince them that the Legion of Doom elite were really +an all-right crew. The premiere of "Phoenix Project" was heavily trumpeted +by Phrack,and "Phoenix Project" carried a complete run of Phrack issues, +including the E911 Document as Phrack had published it. + +Phoenix Project was only one of many--possibly hundreds--of nodes and boards +all over America that were in guilty possession of the E911 Document. +But Phoenix was an outright, unashamed Legion of Doom board. +Under Mentor's guidance, it was flaunting itself in the face +of telco security personnel. Worse yet, it was actively trying +to WIN THEM OVER as sympathizers for the digital underground elite. +"Phoenix" had no cards or codes on it. Its hacker elite considered +Phoenix at least technically legal. But Phoenix was a corrupting influence, +where hacker anarchy was eating away like digital acid at the underbelly +of corporate propriety. + +The Chicago Computer Fraud and Abuse Task Force now prepared +to descend upon Austin, Texas. + +Oddly, not one but TWO trails of the Task Force's investigation led +toward Austin. The city of Austin, like Atlanta, had made itself +a bulwark of the Sunbelt's Information Age, with a strong university +research presence, and a number of cutting-edge electronics companies, +including Motorola, Dell, CompuAdd, IBM, Sematech and MCC. + +Where computing machinery went, hackers generally followed. +Austin boasted not only "Phoenix Project," currently LoD's +most flagrant underground board, but a number of UNIX nodes. + +One of these nodes was "Elephant," run by a UNIX consultant +named Robert Izenberg. Izenberg, in search of a relaxed Southern +lifestyle and a lowered cost-of-living, had recently migrated +to Austin from New Jersey. In New Jersey, Izenberg had worked +for an independent contracting company, programming UNIX code for +AT&T itself. "Terminus" had been a frequent user on Izenberg's +privately owned Elephant node. + +Having interviewed Terminus and examined the records on Netsys, +the Chicago Task Force were now convinced that they had discovered +an underground gang of UNIX software pirates, who were demonstrably +guilty of interstate trafficking in illicitly copied AT&T source code. +Izenberg was swept into the dragnet around Terminus, the self-proclaimed +ultimate UNIX hacker. + +Izenberg, in Austin, had settled down into a UNIX job +with a Texan branch of IBM. Izenberg was no longer +working as a contractor for AT&T, but he had friends +in New Jersey, and he still logged on to AT&T UNIX +computers back in New Jersey, more or less whenever +it pleased him. Izenberg's activities appeared highly +suspicious to the Task Force. Izenberg might well be +breaking into AT&T computers, swiping AT&T software, +and passing it to Terminus and other possible confederates, +through the UNIX node network. And this data was worth, +not merely $79,499, but hundreds of thousands of dollars! + +On February 21, 1990, Robert Izenberg arrived home +from work at IBM to find that all the computers +had mysteriously vanished from his Austin apartment. +Naturally he assumed that he had been robbed. +His "Elephant" node, his other machines, his notebooks, +his disks, his tapes, all gone! However, nothing much +else seemed disturbed--the place had not been ransacked. +The puzzle becaming much stranger some five minutes later. +Austin U. S. Secret Service Agent Al Soliz, accompanied by +University of Texas campus-security officer Larry Coutorie +and the ubiquitous Tim Foley, made their appearance at Izenberg's door. +They were in plain clothes: slacks, polo shirts. They came in, +and Tim Foley accused Izenberg of belonging to the Legion of Doom. + +Izenberg told them that he had never heard of the "Legion of Doom." +And what about a certain stolen E911 Document, that posed a direct +threat to the police emergency lines? Izenberg claimed that he'd +never heard of that, either. + +His interrogators found this difficult to believe. +Didn't he know Terminus? + +Who? + +They gave him Terminus's real name. Oh yes, said Izenberg. +He knew THAT guy all right--he was leading discussions +on the Internet about AT&T computers, especially the AT&T 3B2. + +AT&T had thrust this machine into the marketplace, +but, like many of AT&T's ambitious attempts to enter +the computing arena, the 3B2 project had something less +than a glittering success. Izenberg himself had been +a contractor for the division of AT&T that supported the 3B2. +The entire division had been shut down. + +Nowadays, the cheapest and quickest way to get help with this +fractious piece of machinery was to join one of Terminus's +discussion groups on the Internet, where friendly and knowledgeable +hackers would help you for free. Naturally the remarks within this +group were less than flattering about the Death Star. . .was +THAT the problem? + +Foley told Izenberg that Terminus had been acquiring hot software +through his, Izenberg's, machine. + +Izenberg shrugged this off. A good eight megabytes of data flowed +through his UUCP site every day. UUCP nodes spewed data like fire hoses. +Elephant had been directly linked to Netsys--not surprising, since Terminus +was a 3B2 expert and Izenberg had been a 3B2 contractor. +Izenberg was also linked to "attctc" and the University of Texas. +Terminus was a well-known UNIX expert, and might have been up to +all manner of hijinks on Elephant. Nothing Izenberg could do about that. +That was physically impossible. Needle in a haystack. + +In a four-hour grilling, Foley urged Izenberg to come clean +and admit that he was in conspiracy with Terminus, +and a member of the Legion of Doom. + +Izenberg denied this. He was no weirdo teenage hacker-- +he was thirty-two years old, and didn't even have a "handle." +Izenberg was a former TV technician and electronics specialist +who had drifted into UNIX consulting as a full-grown adult. +Izenberg had never met Terminus, physically. He'd once bought +a cheap high-speed modem from him, though. + +Foley told him that this modem (a Telenet T2500 which ran at 19.2 kilobaud, +and which had just gone out Izenberg's door in Secret Service custody) +was likely hot property. Izenberg was taken aback to hear this; but then +again, most of Izenberg's equipment, like that of most freelance professionals +in the industry, was discounted, passed hand-to-hand through various kinds +of barter and gray-market. There was no proof that the modem was stolen, +and even if it were, Izenberg hardly saw how that gave them the right +to take every electronic item in his house. + +Still, if the United States Secret Service figured they needed +his computer for national security reasons--or whatever-- +then Izenberg would not kick. He figured he would somehow +make the sacrifice of his twenty thousand dollars' worth +of professional equipment, in the spirit of full cooperation +and good citizenship. + +Robert Izenberg was not arrested. Izenberg was not charged with any crime. +His UUCP node--full of some 140 megabytes of the files, mail, and data +of himself and his dozen or so entirely innocent users--went out the door +as "evidence." Along with the disks and tapes, Izenberg had lost about +800 megabytes of data. + +Six months would pass before Izenberg decided to phone the Secret Service +and ask how the case was going. That was the first time that Robert Izenberg +would ever hear the name of William Cook. As of January 1992, a full +two years after the seizure, Izenberg, still not charged with any crime, +would be struggling through the morass of the courts, in hope of recovering +his thousands of dollars' worth of seized equipment. + +In the meantime, the Izenberg case received absolutely no press coverage. +The Secret Service had walked into an Austin home, removed a UNIX bulletin- +board system, and met with no operational difficulties whatsoever. + +Except that word of a crackdown had percolated through the Legion of Doom. +"The Mentor" voluntarily shut down "The Phoenix Project." It seemed a pity, +especially as telco security employees had, in fact, shown up on Phoenix, +just as he had hoped--along with the usual motley crowd of LoD heavies, +hangers-on, phreaks, hackers and wannabes. There was "Sandy" Sandquist from +US SPRINT security, and some guy named Henry Kluepfel, from Bellcore itself! +Kluepfel had been trading friendly banter with hackers on Phoenix since +January 30th (two weeks after the Martin Luther King Day Crash). +The presence of such a stellar telco official seemed quite the coup +for Phoenix Project. + +Still, Mentor could judge the climate. Atlanta in ruins, +Phrack in deep trouble, something weird going on with UNIX nodes-- +discretion was advisable. Phoenix Project went off-line. + +Kluepfel, of course, had been monitoring this LoD bulletin +board for his own purposes--and those of the Chicago unit. +As far back as June 1987, Kluepfel had logged on to a Texas +underground board called "Phreak Klass 2600." There he'd +discovered an Chicago youngster named "Shadowhawk," +strutting and boasting about rifling AT&T computer files, +and bragging of his ambitions to riddle AT&T's Bellcore +computers with trojan horse programs. Kluepfel had passed +the news to Cook in Chicago, Shadowhawk's computers +had gone out the door in Secret Service custody, +and Shadowhawk himself had gone to jail. + +Now it was Phoenix Project's turn. Phoenix Project postured +about "legality" and "merely intellectual interest," but it reeked +of the underground. It had Phrack on it. It had the E911 Document. +It had a lot of dicey talk about breaking into systems, including some +bold and reckless stuff about a supposed "decryption service" that Mentor +and friends were planning to run, to help crack encrypted passwords off +of hacked systems. + +Mentor was an adult. There was a bulletin board at his place of work, +as well. Kleupfel logged onto this board, too, and discovered it to be +called "Illuminati." It was run by some company called Steve Jackson Games. + +On March 1, 1990, the Austin crackdown went into high gear. + +On the morning of March 1--a Thursday--21-year-old University of Texas +student "Erik Bloodaxe," co-sysop of Phoenix Project and an avowed member +of the Legion of Doom, was wakened by a police revolver levelled at his head. + +Bloodaxe watched, jittery, as Secret Service agents +appropriated his 300 baud terminal and, rifling his files, +discovered his treasured source-code for Robert Morris's +notorious Internet Worm. But Bloodaxe, a wily operator, +had suspected that something of the like might be coming. +All his best equipment had been hidden away elsewhere. +The raiders took everything electronic, however, +including his telephone. They were stymied by his +hefty arcade-style Pac-Man game, and left it in place, +as it was simply too heavy to move. + +Bloodaxe was not arrested. He was not charged with any crime. +A good two years later, the police still had what they had +taken from him, however. + +The Mentor was less wary. The dawn raid rousted him and his wife +from bed in their underwear, and six Secret Service agents, +accompanied by an Austin policeman and Henry Kluepfel himself, +made a rich haul. Off went the works, into the agents' white +Chevrolet minivan: an IBM PC-AT clone with 4 meg of RAM and +a 120-meg hard disk; a Hewlett-Packard LaserJet II printer; +a completely legitimate and highly expensive SCO-Xenix 286 +operating system; Pagemaker disks and documentation; +and the Microsoft Word word-processing program. Mentor's wife +had her incomplete academic thesis stored on the hard-disk; +that went, too, and so did the couple's telephone. As of two years later, +all this property remained in police custody. + +Mentor remained under guard in his apartment as agents prepared +to raid Steve Jackson Games. The fact that this was a business +headquarters and not a private residence did not deter the agents. +It was still very early; no one was at work yet. The agents prepared +to break down the door, but Mentor, eavesdropping on the Secret Service +walkie-talkie traffic, begged them not to do it, and offered his key +to the building. + +The exact details of the next events are unclear. The agents +would not let anyone else into the building. Their search warrant, +when produced, was unsigned. Apparently they breakfasted from the local +"Whataburger," as the litter from hamburgers was later found inside. +They also extensively sampled a bag of jellybeans kept by an SJG employee. +Someone tore a "Dukakis for President" sticker from the wall. + +SJG employees, diligently showing up for the day's work, were met +at the door and briefly questioned by U.S. Secret Service agents. +The employees watched in astonishment as agents wielding crowbars +and screwdrivers emerged with captive machines. They attacked +outdoor storage units with boltcutters. The agents wore +blue nylon windbreakers with "SECRET SERVICE" stencilled +across the back, with running-shoes and jeans. + +Jackson's company lost three computers, several hard-disks, +hundred of floppy disks, two monitors, three modems, +a laser printer, various powercords, cables, and adapters +(and, oddly, a small bag of screws, bolts and nuts). +The seizure of Illuminati BBS deprived SJG of all the programs, +text files, and private e-mail on the board. The loss of two other +SJG computers was a severe blow as well, since it caused the loss +of electronically stored contracts, financial projections, +address directories, mailing lists, personnel files, +business correspondence, and, not least, the drafts +of forthcoming games and gaming books. + +No one at Steve Jackson Games was arrested. No one was accused +of any crime. No charges were filed. Everything appropriated +was officially kept as "evidence" of crimes never specified. + +After the Phrack show-trial, the Steve Jackson Games scandal +was the most bizarre and aggravating incident of the Hacker +Crackdown of 1990. This raid by the Chicago Task Force +on a science-fiction gaming publisher was to rouse a +swarming host of civil liberties issues, and gave rise +to an enduring controversy that was still re-complicating itself, +and growing in the scope of its implications, a full two years later. + +The pursuit of the E911 Document stopped with the Steve Jackson Games raid. +As we have seen, there were hundreds, perhaps thousands of computer users +in America with the E911 Document in their possession. Theoretically, +Chicago had a perfect legal right to raid any of these people, +and could have legally seized the machines of anybody who subscribed to Phrack. +However, there was no copy of the E911 Document on Jackson's Illuminati board. +And there the Chicago raiders stopped dead; they have not raided anyone since. + +It might be assumed that Rich Andrews and Charlie Boykin, who had brought +the E911 Document to the attention of telco security, might be spared +any official suspicion. But as we have seen, the willingness to +"cooperate fully" offers little, if any, assurance against federal +anti-hacker prosecution. + +Richard Andrews found himself in deep trouble, thanks to the E911 Document. +Andrews lived in Illinois, the native stomping grounds of the Chicago +Task Force. On February 3 and 6, both his home and his place of work +were raided by USSS. His machines went out the door, too, and he was +grilled at length (though not arrested). Andrews proved to be in +purportedly guilty possession of: UNIX SVR 3.2; UNIX SVR 3.1; UUCP; +PMON; WWB; IWB; DWB; NROFF; KORN SHELL '88; C++; and QUEST, +among other items. Andrews had received this proprietary code-- +which AT&T officially valued at well over $250,000--through the +UNIX network, much of it supplied to him as a personal favor by Terminus. +Perhaps worse yet, Andrews admitted to returning the favor, by passing +Terminus a copy of AT&T proprietary STARLAN source code. + +Even Charles Boykin, himself an AT&T employee, entered some very hot water. +By 1990, he'd almost forgotten about the E911 problem he'd reported in +September 88; in fact, since that date, he'd passed two more security alerts +to Jerry Dalton, concerning matters that Boykin considered far worse than +the E911 Document. + +But by 1990, year of the crackdown, AT&T Corporate Information Security +was fed up with "Killer." This machine offered no direct income to AT&T, +and was providing aid and comfort to a cloud of suspicious yokels +from outside the company, some of them actively malicious toward AT&T, +its property, and its corporate interests. Whatever goodwill and publicity +had been won among Killer's 1,500 devoted users was considered no longer +worth the security risk. On February 20, 1990, Jerry Dalton arrived in +Dallas and simply unplugged the phone jacks, to the puzzled alarm +of Killer's many Texan users. Killer went permanently off-line, +with the loss of vast archives of programs and huge quantities +of electronic mail; it was never restored to service. AT&T showed +no particular regard for the "property" of these 1,500 people. +Whatever "property" the users had been storing on AT&T's computer +simply vanished completely. + +Boykin, who had himself reported the E911 problem, +now found himself under a cloud of suspicion. In a weird +private-security replay of the Secret Service seizures, +Boykin's own home was visited by AT&T Security and his +own machines were carried out the door. + +However, there were marked special features in the Boykin case. +Boykin's disks and his personal computers were swiftly examined +by his corporate employers and returned politely in just two days-- +(unlike Secret Service seizures, which commonly take months or years). +Boykin was not charged with any crime or wrongdoing, and he kept his job +with AT&T (though he did retire from AT&T in September 1991, +at the age of 52). + +It's interesting to note that the US Secret Service somehow failed +to seize Boykin's "Killer" node and carry AT&T's own computer out the door. +Nor did they raid Boykin's home. They seemed perfectly willing to take the +word of AT&T Security that AT&T's employee, and AT&T's "Killer" node, +were free of hacker contraband and on the up-and-up. + +It's digital water-under-the-bridge at this point, as Killer's +3,200 megabytes of Texan electronic community were erased in 1990, +and "Killer" itself was shipped out of the state. + +But the experiences of Andrews and Boykin, and the users of their systems, +remained side issues. They did not begin to assume the social, political, +and legal importance that gathered, slowly but inexorably, around the issue +of the raid on Steve Jackson Games. + +# + +We must now turn our attention to Steve Jackson Games itself, +and explain what SJG was, what it really did, and how it had +managed to attract this particularly odd and virulent kind of trouble. +The reader may recall that this is not the first but the second time +that the company has appeared in this narrative; a Steve Jackson game +called GURPS was a favorite pastime of Atlanta hacker Urvile, +and Urvile's science-fictional gaming notes had been mixed up +promiscuously with notes about his actual computer intrusions. + +First, Steve Jackson Games, Inc., was NOT a publisher of "computer games." +SJG published "simulation games," parlor games that were played on paper, +with pencils, and dice, and printed guidebooks full of rules and +statistics tables. There were no computers involved in the games themselves. +When you bought a Steve Jackson Game, you did not receive any software disks. +What you got was a plastic bag with some cardboard game tokens, +maybe a few maps or a deck of cards. Most of their products were books. + +However, computers WERE deeply involved in the Steve Jackson Games business. +Like almost all modern publishers, Steve Jackson and his fifteen employees +used computers to write text, to keep accounts, and to run the business +generally. They also used a computer to run their official bulletin board +system for Steve Jackson Games, a board called Illuminati. On Illuminati, +simulation gamers who happened to own computers and modems could associate, +trade mail, debate the theory and practice of gaming, and keep up with the +company's news and its product announcements. + +Illuminati was a modestly popular board, run on a small computer +with limited storage, only one phone-line, and no ties to large-scale +computer networks. It did, however, have hundreds of users, +many of them dedicated gamers willing to call from out-of-state. + +Illuminati was NOT an "underground" board. It did not feature hints +on computer intrusion, or "anarchy files," or illicitly posted +credit card numbers, or long-distance access codes. +Some of Illuminati's users, however, were members of the Legion of Doom. +And so was one of Steve Jackson's senior employees--the Mentor. +The Mentor wrote for Phrack, and also ran an underground board, +Phoenix Project--but the Mentor was not a computer professional. +The Mentor was the managing editor of Steve Jackson Games and +a professional game designer by trade. These LoD members did not +use Illuminati to help their HACKING activities. They used it to +help their GAME-PLAYING activities--and they were even more dedicated +to simulation gaming than they were to hacking. + +"Illuminati" got its name from a card-game that Steve Jackson himself, +the company's founder and sole owner, had invented. This multi-player +card-game was one of Mr Jackson's best-known, most successful, +most technically innovative products. "Illuminati" was a game +of paranoiac conspiracy in which various antisocial cults warred +covertly to dominate the world. "Illuminati" was hilarious, +and great fun to play, involving flying saucers, the CIA, the KGB, +the phone companies, the Ku Klux Klan, the South American Nazis, +the cocaine cartels, the Boy Scouts, and dozens of other splinter groups +from the twisted depths of Mr. Jackson's professionally fervid imagination. +For the uninitiated, any public discussion of the "Illuminati" card-game +sounded, by turns, utterly menacing or completely insane. + +And then there was SJG's "Car Wars," in which souped-up armored hot-rods +with rocket-launchers and heavy machine-guns did battle on the American +highways of the future. The lively Car Wars discussion on the Illuminati +board featured many meticulous, painstaking discussions of the effects +of grenades, land-mines, flamethrowers and napalm. It sounded like +hacker anarchy files run amuck. + +Mr Jackson and his co-workers earned their daily bread by supplying people +with make-believe adventures and weird ideas. The more far-out, the better. + +Simulation gaming is an unusual pastime, but gamers have not +generally had to beg the permission of the Secret Service to exist. +Wargames and role-playing adventures are an old and honored pastime, +much favored by professional military strategists. Once little-known, +these games are now played by hundreds of thousands of enthusiasts +throughout North America, Europe and Japan. Gaming-books, once restricted +to hobby outlets, now commonly appear in chain-stores like B. Dalton's +and Waldenbooks, and sell vigorously. + +Steve Jackson Games, Inc., of Austin, Texas, was a games company +of the middle rank. In 1989, SJG grossed about a million dollars. +Jackson himself had a good reputation in his industry as a talented +and innovative designer of rather unconventional games, but his company +was something less than a titan of the field--certainly not like the +multimillion-dollar TSR Inc., or Britain's gigantic "Games Workshop." +SJG's Austin headquarters was a modest two-story brick office-suite, +cluttered with phones, photocopiers, fax machines and computers. +It bustled with semi-organized activity and was littered with +glossy promotional brochures and dog-eared science-fiction novels. +Attached to the offices was a large tin-roofed warehouse piled twenty feet +high with cardboard boxes of games and books. Despite the weird imaginings +that went on within it, the SJG headquarters was quite a quotidian, +everyday sort of place. It looked like what it was: a publishers' digs. + +Both "Car Wars" and "Illuminati" were well-known, popular games. +But the mainstay of the Jackson organization was their Generic Universal +Role-Playing System, "G.U.R.P.S." The GURPS system was considered solid +and well-designed, an asset for players. But perhaps the most popular +feature of the GURPS system was that it allowed gaming-masters to design +scenarios that closely resembled well-known books, movies, and other works +of fantasy. Jackson had licensed and adapted works from many science fiction +and fantasy authors. There was GURPS Conan, GURPS Riverworld, +GURPS Horseclans, GURPS Witch World, names eminently familiar +to science-fiction readers. And there was GURPS Special Ops, +from the world of espionage fantasy and unconventional warfare. + +And then there was GURPS Cyberpunk. + +"Cyberpunk" was a term given to certain science fiction writers +who had entered the genre in the 1980s. "Cyberpunk," as the label implies, +had two general distinguishing features. First, its writers had a compelling +interest in information technology, an interest closely akin +to science fiction's earlier fascination with space travel. +And second, these writers were "punks," with all the +distinguishing features that that implies: Bohemian artiness, +youth run wild, an air of deliberate rebellion, funny clothes and hair, +odd politics, a fondness for abrasive rock and roll; in a word, trouble. + +The "cyberpunk" SF writers were a small group of mostly college-educated +white middle-class litterateurs, scattered through the US and Canada. +Only one, Rudy Rucker, a professor of computer science in Silicon Valley, +could rank with even the humblest computer hacker. But, except for +Professor Rucker, the "cyberpunk" authors were not programmers +or hardware experts; they considered themselves artists +(as, indeed, did Professor Rucker). However, these writers +all owned computers, and took an intense and public interest +in the social ramifications of the information industry. + +The cyberpunks had a strong following among the global generation +that had grown up in a world of computers, multinational networks, +and cable television. Their outlook was considered somewhat morbid, +cynical, and dark, but then again, so was the outlook of their +generational peers. As that generation matured and increased +in strength and influence, so did the cyberpunks. +As science-fiction writers went, they were doing +fairly well for themselves. By the late 1980s, +their work had attracted attention from gaming companies, +including Steve Jackson Games, which was planning a cyberpunk +simulation for the flourishing GURPS gaming-system. + +The time seemed ripe for such a product, which had already been proven +in the marketplace. The first games- company out of the gate, +with a product boldly called "Cyberpunk" in defiance of possible +infringement-of-copyright suits, had been an upstart group called +R. Talsorian. Talsorian's Cyberpunk was a fairly decent game, +but the mechanics of the simulation system left a lot to be desired. +Commercially, however, the game did very well. + +The next cyberpunk game had been the even more successful Shadowrun +by FASA Corporation. The mechanics of this game were fine, but the +scenario was rendered moronic by sappy fantasy elements like elves, +trolls, wizards, and dragons--all highly ideologically-incorrect, +according to the hard-edged, high-tech standards of cyberpunk science fiction. + +Other game designers were champing at the bit. Prominent among them +was the Mentor, a gentleman who, like most of his friends in the +Legion of Doom, was quite the cyberpunk devotee. Mentor reasoned +that the time had come for a REAL cyberpunk gaming-book--one that the +princes of computer-mischief in the Legion of Doom could play without +laughing themselves sick. This book, GURPS Cyberpunk, would reek +of culturally on-line authenticity. + +Mentor was particularly well-qualified for this task. +Naturally, he knew far more about computer-intrusion +and digital skullduggery than any previously published +cyberpunk author. Not only that, but he was good at his work. +A vivid imagination, combined with an instinctive feeling +for the working of systems and, especially, the loopholes +within them, are excellent qualities for a professional game designer. + +By March 1st, GURPS Cyberpunk was almost complete, ready to print and ship. +Steve Jackson expected vigorous sales for this item, which, he hoped, +would keep the company financially afloat for several months. +GURPS Cyberpunk, like the other GURPS "modules," was not a "game" +like a Monopoly set, but a BOOK: a bound paperback book the size +of a glossy magazine, with a slick color cover, and pages full of text, +illustrations, tables and footnotes. It was advertised as a game, +and was used as an aid to game-playing, but it was a book, +with an ISBN number, published in Texas, copyrighted, +and sold in bookstores. + +And now, that book, stored on a computer, had gone out the door +in the custody of the Secret Service. + +The day after the raid, Steve Jackson visited the local Secret Service +headquarters with a lawyer in tow. There he confronted Tim Foley +(still in Austin at that time) and demanded his book back. But there +was trouble. GURPS Cyberpunk, alleged a Secret Service agent to astonished +businessman Steve Jackson, was "a manual for computer crime." + +"It's science fiction," Jackson said. + +"No, this is real." + +This statement was repeated several times, by several agents. +Jackson's ominously accurate game had passed from pure, +obscure, small-scale fantasy into the impure, highly publicized, +large-scale fantasy of the Hacker Crackdown. + +No mention was made of the real reason for the search. +According to their search warrant, the raiders had expected +to find the E911 Document stored on Jackson's bulletin board system. +But that warrant was sealed; a procedure that most law enforcement agencies +will use only when lives are demonstrably in danger. The raiders' +true motives were not discovered until the Jackson search-warrant +was unsealed by his lawyers, many months later. The Secret Service, +and the Chicago Computer Fraud and Abuse Task Force, +said absolutely nothing to Steve Jackson about any threat +to the police 911 System. They said nothing about the Atlanta Three, +nothing about Phrack or Knight Lightning, nothing about Terminus. + +Jackson was left to believe that his computers had been seized because +he intended to publish a science fiction book that law enforcement +considered too dangerous to see print. + +This misconception was repeated again and again, for months, +to an ever-widening public audience. It was not the truth of the case; +but as months passed, and this misconception was publicly printed again +and again, it became one of the few publicly known "facts" about +the mysterious Hacker Crackdown. The Secret Service had seized a computer +to stop the publication of a cyberpunk science fiction book. + +The second section of this book, "The Digital Underground," +is almost finished now. We have become acquainted with all +the major figures of this case who actually belong to the +underground milieu of computer intrusion. We have some idea +of their history, their motives, their general modus operandi. +We now know, I hope, who they are, where they came from, +and more or less what they want. In the next section of this book, +"Law and Order," we will leave this milieu and directly enter the +world of America's computer-crime police. + +At this point, however, I have another figure to introduce: myself. + +My name is Bruce Sterling. I live in Austin, Texas, where I am +a science fiction writer by trade: specifically, a CYBERPUNK +science fiction writer. + +Like my "cyberpunk" colleagues in the U.S. and Canada, +I've never been entirely happy with this literary label-- +especially after it became a synonym for computer criminal. +But I did once edit a book of stories by my colleagues, +called Mirrorshades: the Cyberpunk Anthology, and I've +long been a writer of literary-critical cyberpunk manifestos. +I am not a "hacker" of any description, though I do have readers +in the digital underground. + +When the Steve Jackson Games seizure occurred, I naturally took +an intense interest. If "cyberpunk" books were being banned +by federal police in my own home town, I reasonably wondered +whether I myself might be next. Would my computer be seized +by the Secret Service? At the time, I was in possession +of an aging Apple IIe without so much as a hard disk. +If I were to be raided as an author of computer-crime manuals, +the loss of my feeble word-processor would likely provoke more +snickers than sympathy. + +I'd known Steve Jackson for many years. We knew +one another as colleagues, for we frequented +the same local science-fiction conventions. +I'd played Jackson games, and recognized his cleverness; +but he certainly had never struck me as a potential mastermind +of computer crime. + +I also knew a little about computer bulletin-board systems. +In the mid-1980s I had taken an active role in an Austin board +called "SMOF-BBS," one of the first boards dedicated to science fiction. +I had a modem, and on occasion I'd logged on to Illuminati, +which always looked entertainly wacky, but certainly harmless enough. + +At the time of the Jackson seizure, I had no experience +whatsoever with underground boards. But I knew that no one +on Illuminati talked about breaking into systems illegally, +or about robbing phone companies. Illuminati didn't even +offer pirated computer games. Steve Jackson, like many creative artists, +was markedly touchy about theft of intellectual property. + +It seemed to me that Jackson was either seriously suspected +of some crime--in which case, he would be charged soon, +and would have his day in court--or else he was innocent, +in which case the Secret Service would quickly return his equipment, +and everyone would have a good laugh. I rather expected the good laugh. +The situation was not without its comic side. The raid, known +as the "Cyberpunk Bust" in the science fiction community, +was winning a great deal of free national publicity both +for Jackson himself and the "cyberpunk" science fiction +writers generally. + +Besides, science fiction people are used to being misinterpreted. +Science fiction is a colorful, disreputable, slipshod occupation, +full of unlikely oddballs, which, of course, is why we like it. +Weirdness can be an occupational hazard in our field. People who +wear Halloween costumes are sometimes mistaken for monsters. + +Once upon a time--back in 1939, in New York City-- +science fiction and the U.S. Secret Service collided in +a comic case of mistaken identity. This weird incident +involved a literary group quite famous in science fiction, +known as "the Futurians," whose membership included +such future genre greats as Isaac Asimov, Frederik Pohl, +and Damon Knight. The Futurians were every bit as +offbeat and wacky as any of their spiritual descendants, +including the cyberpunks, and were given to communal living, +spontaneous group renditions of light opera, and midnight fencing +exhibitions on the lawn. The Futurians didn't have bulletin +board systems, but they did have the technological equivalent +in 1939--mimeographs and a private printing press. These were +in steady use, producing a stream of science-fiction fan magazines, +literary manifestos, and weird articles, which were picked up +in ink-sticky bundles by a succession of strange, gangly, +spotty young men in fedoras and overcoats. + +The neighbors grew alarmed at the antics of the Futurians +and reported them to the Secret Service as suspected counterfeiters. +In the winter of 1939, a squad of USSS agents with drawn guns burst into +"Futurian House," prepared to confiscate the forged currency and illicit +printing presses. There they discovered a slumbering science fiction fan +named George Hahn, a guest of the Futurian commune who had just arrived +in New York. George Hahn managed to explain himself and his group, +and the Secret Service agents left the Futurians in peace henceforth. +(Alas, Hahn died in 1991, just before I had discovered this astonishing +historical parallel, and just before I could interview him for this book.) + +But the Jackson case did not come to a swift and comic end. +No quick answers came his way, or mine; no swift reassurances +that all was right in the digital world, that matters were well +in hand after all. Quite the opposite. In my alternate role +as a sometime pop-science journalist, I interviewed Jackson +and his staff for an article in a British magazine. +The strange details of the raid left me more concerned than ever. +Without its computers, the company had been financially +and operationally crippled. Half the SJG workforce, +a group of entirely innocent people, had been sorrowfully fired, +deprived of their livelihoods by the seizure. It began to dawn on me +that authors--American writers--might well have their computers seized, +under sealed warrants, without any criminal charge; and that, +as Steve Jackson had discovered, there was no immediate recourse for this. +This was no joke; this wasn't science fiction; this was real. + +I determined to put science fiction aside until I had discovered +what had happened and where this trouble had come from. +It was time to enter the purportedly real world of electronic +free expression and computer crime. Hence, this book. +Hence, the world of the telcos; and the world of the digital underground; +and next, the world of the police. + + + +PART THREE: LAW AND ORDER + + +Of the various anti-hacker activities of 1990, "Operation Sundevil" +had by far the highest public profile. The sweeping, nationwide +computer seizures of May 8, 1990 were unprecedented in scope and highly, +if rather selectively, publicized. + +Unlike the efforts of the Chicago Computer Fraud and Abuse Task Force, +"Operation Sundevil" was not intended to combat "hacking" in the sense +of computer intrusion or sophisticated raids on telco switching stations. +Nor did it have anything to do with hacker misdeeds with AT&T's software, +or with Southern Bell's proprietary documents. + +Instead, "Operation Sundevil" was a crackdown on those traditional scourges +of the digital underground: credit-card theft and telephone code abuse. +The ambitious activities out of Chicago, and the somewhat lesser-known +but vigorous anti-hacker actions of the New York State Police in 1990, +were never a part of "Operation Sundevil" per se, which was based in Arizona. + +Nevertheless, after the spectacular May 8 raids, the public, misled by +police secrecy, hacker panic, and a puzzled national press-corps, +conflated all aspects of the nationwide crackdown in 1990 under +the blanket term "Operation Sundevil." "Sundevil" is still the best-known +synonym for the crackdown of 1990. But the Arizona organizers of "Sundevil" +did not really deserve this reputation--any more, for instance, than all +hackers deserve a reputation as "hackers." + +There was some justice in this confused perception, though. +For one thing, the confusion was abetted by the Washington office +of the Secret Service, who responded to Freedom of Information Act +requests on "Operation Sundevil" by referring investigators +to the publicly known cases of Knight Lightning and the Atlanta Three. +And "Sundevil" was certainly the largest aspect of the Crackdown, +the most deliberate and the best-organized. As a crackdown on electronic +fraud, "Sundevil" lacked the frantic pace of the war on the Legion of Doom; +on the contrary, Sundevil's targets were picked out with cool deliberation +over an elaborate investigation lasting two full years. + +And once again the targets were bulletin board systems. + +Boards can be powerful aids to organized fraud. Underground boards carry +lively, extensive, detailed, and often quite flagrant "discussions" of +lawbreaking techniques and lawbreaking activities. "Discussing" crime +in the abstract, or "discussing" the particulars of criminal cases, +is not illegal--but there are stern state and federal laws against +coldbloodedly conspiring in groups in order to commit crimes. + +In the eyes of police, people who actively conspire to break the law +are not regarded as "clubs," "debating salons," "users' groups," or +"free speech advocates." Rather, such people tend to find themselves +formally indicted by prosecutors as "gangs," "racketeers," "corrupt +organizations" and "organized crime figures." + +What's more, the illicit data contained on outlaw boards goes well beyond +mere acts of speech and/or possible criminal conspiracy. As we have seen, +it was common practice in the digital underground to post purloined telephone +codes on boards, for any phreak or hacker who cared to abuse them. Is posting +digital booty of this sort supposed to be protected by the First Amendment? +Hardly--though the issue, like most issues in cyberspace, is not entirely +resolved. Some theorists argue that to merely RECITE a number publicly +is not illegal--only its USE is illegal. But anti-hacker police point out +that magazines and newspapers (more traditional forms of free expression) +never publish stolen telephone codes (even though this might well +raise their circulation). + +Stolen credit card numbers, being riskier and more valuable, +were less often publicly posted on boards--but there is no question +that some underground boards carried "carding" traffic, +generally exchanged through private mail. + +Underground boards also carried handy programs for "scanning" telephone +codes and raiding credit card companies, as well as the usual obnoxious +galaxy of pirated software, cracked passwords, blue-box schematics, +intrusion manuals, anarchy files, porn files, and so forth. + +But besides their nuisance potential for the spread of illicit knowledge, +bulletin boards have another vitally interesting aspect for the +professional investigator. Bulletin boards are cram-full of EVIDENCE. +All that busy trading of electronic mail, all those hacker boasts, +brags and struts, even the stolen codes and cards, can be neat, +electronic, real-time recordings of criminal activity. +As an investigator, when you seize a pirate board, you have +scored a coup as effective as tapping phones or intercepting mail. +However, you have not actually tapped a phone or intercepted a letter. +The rules of evidence regarding phone-taps and mail interceptions are old, +stern and well-understood by police, prosecutors and defense attorneys alike. +The rules of evidence regarding boards are new, waffling, and understood +by nobody at all. + +Sundevil was the largest crackdown on boards in world history. +On May 7, 8, and 9, 1990, about forty-two computer systems were seized. +Of those forty-two computers, about twenty-five actually were running boards. +(The vagueness of this estimate is attributable to the vagueness of +(a) what a "computer system" is, and (b) what it actually means to +"run a board" with one--or with two computers, or with three.) + +About twenty-five boards vanished into police custody in May 1990. +As we have seen, there are an estimated 30,000 boards in America today. +If we assume that one board in a hundred is up to no good with codes +and cards (which rather flatters the honesty of the board-using community), +then that would leave 2,975 outlaw boards untouched by Sundevil. +Sundevil seized about one tenth of one percent of all computer +bulletin boards in America. Seen objectively, this is something less +than a comprehensive assault. In 1990, Sundevil's organizers-- +the team at the Phoenix Secret Service office, and the Arizona +Attorney General's office-- had a list of at least THREE HUNDRED +boards that they considered fully deserving of search and seizure warrants. +The twenty-five boards actually seized were merely among the most obvious +and egregious of this much larger list of candidates. All these boards +had been examined beforehand--either by informants, who had passed printouts +to the Secret Service, or by Secret Service agents themselves, who not only +come equipped with modems but know how to use them. + +There were a number of motives for Sundevil. First, it offered +a chance to get ahead of the curve on wire-fraud crimes. +Tracking back credit-card ripoffs to their perpetrators +can be appallingly difficult. If these miscreants +have any kind of electronic sophistication, they can snarl +their tracks through the phone network into a mind-boggling, +untraceable mess, while still managing to "reach out and rob someone." +Boards, however, full of brags and boasts, codes and cards, +offer evidence in the handy congealed form. + +Seizures themselves--the mere physical removal of machines-- +tends to take the pressure off. During Sundevil, a large number +of code kids, warez d00dz, and credit card thieves would be deprived +of those boards--their means of community and conspiracy--in one swift blow. +As for the sysops themselves (commonly among the boldest offenders) +they would be directly stripped of their computer equipment, +and rendered digitally mute and blind. + +And this aspect of Sundevil was carried out with great success. +Sundevil seems to have been a complete tactical surprise-- +unlike the fragmentary and continuing seizures of the war on the +Legion of Doom, Sundevil was precisely timed and utterly overwhelming. +At least forty "computers" were seized during May 7, 8 and 9, 1990, +in Cincinnati, Detroit, Los Angeles, Miami, Newark, Phoenix, Tucson, +Richmond, San Diego, San Jose, Pittsburgh and San Francisco. +Some cities saw multiple raids, such as the five separate raids +in the New York City environs. Plano, Texas (essentially a suburb of +the Dallas/Fort Worth metroplex, and a hub of the telecommunications industry) +saw four computer seizures. Chicago, ever in the forefront, saw its own +local Sundevil raid, briskly carried out by Secret Service agents +Timothy Foley and Barbara Golden. + +Many of these raids occurred, not in the cities proper, +but in associated white-middle class suburbs--places like +Mount Lebanon, Pennsylvania and Clark Lake, Michigan. +There were a few raids on offices; most took place in people's homes, +the classic hacker basements and bedrooms. + +The Sundevil raids were searches and seizures, not a group of mass arrests. +There were only four arrests during Sundevil. "Tony the Trashman," +a longtime teenage bete noire of the Arizona Racketeering unit, +was arrested in Tucson on May 9. "Dr. Ripco," sysop of an outlaw board +with the misfortune to exist in Chicago itself, was also arrested-- +on illegal weapons charges. Local units also arrested a 19-year-old +female phone phreak named "Electra" in Pennsylvania, and a male juvenile +in California. Federal agents however were not seeking arrests, but computers. + +Hackers are generally not indicted (if at all) until the evidence +in their seized computers is evaluated--a process that can take weeks, +months--even years. When hackers are arrested on the spot, it's generally +an arrest for other reasons. Drugs and/or illegal weapons show up in a good +third of anti-hacker computer seizures (though not during Sundevil). + +That scofflaw teenage hackers (or their parents) should have marijuana +in their homes is probably not a shocking revelation, but the surprisingly +common presence of illegal firearms in hacker dens is a bit disquieting. +A Personal Computer can be a great equalizer for the techno-cowboy-- +much like that more traditional American "Great Equalizer," +the Personal Sixgun. Maybe it's not all that surprising +that some guy obsessed with power through illicit technology +would also have a few illicit high-velocity-impact devices around. +An element of the digital underground particularly dotes on those +"anarchy philes," and this element tends to shade into the crackpot milieu +of survivalists, gun-nuts, anarcho-leftists and the ultra-libertarian +right-wing. + +This is not to say that hacker raids to date have uncovered any +major crack-dens or illegal arsenals; but Secret Service agents +do not regard "hackers" as "just kids." They regard hackers as +unpredictable people, bright and slippery. It doesn't help matters +that the hacker himself has been "hiding behind his keyboard" +all this time. Commonly, police have no idea what he looks like. +This makes him an unknown quantity, someone best treated with +proper caution. + +To date, no hacker has come out shooting, though they do sometimes brag on +boards that they will do just that. Threats of this sort are taken seriously. +Secret Service hacker raids tend to be swift, comprehensive, well-manned +(even over-manned); and agents generally burst through every door +in the home at once, sometimes with drawn guns. Any potential resistance +is swiftly quelled. Hacker raids are usually raids on people's homes. +It can be a very dangerous business to raid an American home; +people can panic when strangers invade their sanctum. Statistically speaking, +the most dangerous thing a policeman can do is to enter someone's home. +(The second most dangerous thing is to stop a car in traffic.) +People have guns in their homes. More cops are hurt in homes +than are ever hurt in biker bars or massage parlors. + +But in any case, no one was hurt during Sundevil, +or indeed during any part of the Hacker Crackdown. + +Nor were there any allegations of any physical mistreatment of a suspect. +Guns were pointed, interrogations were sharp and prolonged; but no one +in 1990 claimed any act of brutality by any crackdown raider. + +In addition to the forty or so computers, Sundevil reaped floppy disks +in particularly great abundance--an estimated 23,000 of them, which +naturally included every manner of illegitimate data: pirated games, +stolen codes, hot credit card numbers, the complete text and software +of entire pirate bulletin-boards. These floppy disks, which remain +in police custody today, offer a gigantic, almost embarrassingly +rich source of possible criminal indictments. These 23,000 floppy disks +also include a thus-far unknown quantity of legitimate computer games, +legitimate software, purportedly "private" mail from boards, +business records, and personal correspondence of all kinds. + +Standard computer-crime search warrants lay great emphasis on seizing +written documents as well as computers--specifically including photocopies, +computer printouts, telephone bills, address books, logs, notes, +memoranda and correspondence. In practice, this has meant that diaries, +gaming magazines, software documentation, nonfiction books on hacking +and computer security, sometimes even science fiction novels, have all +vanished out the door in police custody. A wide variety of electronic items +have been known to vanish as well, including telephones, televisions, answering +machines, Sony Walkmans, desktop printers, compact disks, and audiotapes. + +No fewer than 150 members of the Secret Service were sent into +the field during Sundevil. They were commonly accompanied by +squads of local and/or state police. Most of these officers-- +especially the locals--had never been on an anti-hacker raid before. +(This was one good reason, in fact, why so many of them were invited along +in the first place.) Also, the presence of a uniformed police officer +assures the raidees that the people entering their homes are, in fact, police. +Secret Service agents wear plain clothes. So do the telco security experts +who commonly accompany the Secret Service on raids (and who make no particular +effort to identify themselves as mere employees of telephone companies). + +A typical hacker raid goes something like this. First, police storm in +rapidly, through every entrance, with overwhelming force, +in the assumption that this tactic will keep casualties to a minimum. +Second, possible suspects are immediately removed from the vicinity +of any and all computer systems, so that they will have no chance +to purge or destroy computer evidence. Suspects are herded into a room +without computers, commonly the living room, and kept under guard-- +not ARMED guard, for the guns are swiftly holstered, but under guard +nevertheless. They are presented with the search warrant and warned +that anything they say may be held against them. Commonly they have +a great deal to say, especially if they are unsuspecting parents. + +Somewhere in the house is the "hot spot"--a computer tied to a phone +line (possibly several computers and several phones). Commonly it's +a teenager's bedroom, but it can be anywhere in the house; +there may be several such rooms. This "hot spot" is put in charge +of a two-agent team, the "finder" and the "recorder." The "finder" +is computer-trained, commonly the case agent who has actually obtained +the search warrant from a judge. He or she understands what is being sought, +and actually carries out the seizures: unplugs machines, opens drawers, +desks, files, floppy-disk containers, etc. The "recorder" photographs +all the equipment, just as it stands--especially the tangle of +wired connections in the back, which can otherwise be a real nightmare +to restore. The recorder will also commonly photograph every room +in the house, lest some wily criminal claim that the police had robbed him +during the search. Some recorders carry videocams or tape recorders; +however, it's more common for the recorder to simply take written notes. +Objects are described and numbered as the finder seizes them, generally +on standard preprinted police inventory forms. + +Even Secret Service agents were not, and are not, expert computer users. +They have not made, and do not make, judgements on the fly about potential +threats posed by various forms of equipment. They may exercise discretion; +they may leave Dad his computer, for instance, but they don't HAVE to. +Standard computer-crime search warrants, which date back to the early 80s, +use a sweeping language that targets computers, most anything attached +to a computer, most anything used to operate a computer--most anything +that remotely resembles a computer--plus most any and all written documents +surrounding it. Computer-crime investigators have strongly urged agents +to seize the works. + +In this sense, Operation Sundevil appears to have been a complete success. +Boards went down all over America, and were shipped en masse to the computer +investigation lab of the Secret Service, in Washington DC, along with the +23,000 floppy disks and unknown quantities of printed material. + +But the seizure of twenty-five boards, and the multi-megabyte mountains +of possibly useful evidence contained in these boards (and in their owners' +other computers, also out the door), were far from the only motives for +Operation Sundevil. An unprecedented action of great ambition and size, +Sundevil's motives can only be described as political. It was a +public-relations effort, meant to pass certain messages, meant to make +certain situations clear: both in the mind of the general public, +and in the minds of various constituencies of the electronic community. + + First --and this motivation was vital--a "message" would be sent from +law enforcement to the digital underground. This very message was recited +in so many words by Garry M. Jenkins, the Assistant Director of the +US Secret Service, at the Sundevil press conference in Phoenix on +May 9, 1990, immediately after the raids. In brief, hackers were +mistaken in their foolish belief that they could hide behind the +"relative anonymity of their computer terminals." On the contrary, +they should fully understand that state and federal cops were +actively patrolling the beat in cyberspace--that they were +on the watch everywhere, even in those sleazy and secretive +dens of cybernetic vice, the underground boards. + +This is not an unusual message for police to publicly convey to crooks. +The message is a standard message; only the context is new. + +In this respect, the Sundevil raids were the digital equivalent +of the standard vice-squad crackdown on massage parlors, porno bookstores, +head-shops, or floating crap-games. There may be few or no arrests in a raid +of this sort; no convictions, no trials, no interrogations. In cases of this +sort, police may well walk out the door with many pounds of sleazy magazines, +X-rated videotapes, sex toys, gambling equipment, baggies of marijuana. . . . + +Of course, if something truly horrendous is discovered by the raiders, +there will be arrests and prosecutions. Far more likely, however, +there will simply be a brief but sharp disruption of the closed +and secretive world of the nogoodniks. There will be "street hassle." +"Heat." "Deterrence." And, of course, the immediate loss of the seized goods. +It is very unlikely that any of this seized material will ever be returned. +Whether charged or not, whether convicted or not, the perpetrators will +almost surely lack the nerve ever to ask for this stuff to be given back. + +Arrests and trials--putting people in jail--may involve all kinds of +formal legalities; but dealing with the justice system is far from the only +task of police. Police do not simply arrest people. They don't simply +put people in jail. That is not how the police perceive their jobs. +Police "protect and serve." Police "keep the peace," they "keep public order." +Like other forms of public relations, keeping public order is not an +exact science. Keeping public order is something of an art-form. + +If a group of tough-looking teenage hoodlums was loitering on a street-corner, +no one would be surprised to see a street-cop arrive and sternly order +them to "break it up." On the contrary, the surprise would come if one +of these ne'er-do-wells stepped briskly into a phone-booth, +called a civil rights lawyer, and instituted a civil suit +in defense of his Constitutional rights of free speech +and free assembly. But something much along this line +was one of the many anomolous outcomes of the Hacker Crackdown. + +Sundevil also carried useful "messages" for other constituents of +the electronic community. These messages may not have been read +aloud from the Phoenix podium in front of the press corps, +but there was little mistaking their meaning. There was a message +of reassurance for the primary victims of coding and carding: +the telcos, and the credit companies. Sundevil was greeted with joy +by the security officers of the electronic business community. +After years of high-tech harassment and spiralling revenue losses, +their complaints of rampant outlawry were being taken seriously by +law enforcement. No more head-scratching or dismissive shrugs; +no more feeble excuses about "lack of computer-trained officers" or +the low priority of "victimless" white-collar telecommunication crimes. + +Computer-crime experts have long believed that computer-related offenses +are drastically under-reported. They regard this as a major open scandal +of their field. Some victims are reluctant to come forth, because they +believe that police and prosecutors are not computer-literate, +and can and will do nothing. Others are embarrassed by +their vulnerabilities, and will take strong measures +to avoid any publicity; this is especially true of banks, +who fear a loss of investor confidence should an embezzlement-case +or wire-fraud surface. And some victims are so helplessly confused +by their own high technology that they never even realize that +a crime has occurred--even when they have been fleeced to the bone. + +The results of this situation can be dire. +Criminals escape apprehension and punishment. +The computer-crime units that do exist, can't get work. +The true scope of computer-crime: its size, its real nature, +the scope of its threats, and the legal remedies for it-- +all remain obscured. + +Another problem is very little publicized, but it is a cause +of genuine concern. Where there is persistent crime, +but no effective police protection, then vigilantism can result. +Telcos, banks, credit companies, the major corporations who +maintain extensive computer networks vulnerable to hacking +--these organizations are powerful, wealthy, and +politically influential. They are disinclined to be +pushed around by crooks (or by most anyone else, +for that matter). They often maintain well-organized +private security forces, commonly run by +experienced veterans of military and police units, +who have left public service for the greener pastures +of the private sector. For police, the corporate +security manager can be a powerful ally; but if this +gentleman finds no allies in the police, and the +pressure is on from his board-of-directors, +he may quietly take certain matters into his own hands. + +Nor is there any lack of disposable hired-help in the +corporate security business. Private security agencies-- +the `security business' generally--grew explosively in the 1980s. +Today there are spooky gumshoed armies of "security consultants," +"rent-a- cops," "private eyes," "outside experts"--every manner +of shady operator who retails in "results" and discretion. +Or course, many of these gentlemen and ladies may be paragons +of professional and moral rectitude. But as anyone +who has read a hard-boiled detective novel knows, +police tend to be less than fond of this sort +of private-sector competition. + +Companies in search of computer-security have even been +known to hire hackers. Police shudder at this prospect. + +Police treasure good relations with the business community. +Rarely will you see a policeman so indiscreet as to allege +publicly that some major employer in his state or city has succumbed +to paranoia and gone off the rails. Nevertheless, +police --and computer police in particular--are aware +of this possibility. Computer-crime police can and do +spend up to half of their business hours just doing +public relations: seminars, "dog and pony shows," +sometimes with parents' groups or computer users, +but generally with their core audience: the likely +victims of hacking crimes. These, of course, are telcos, +credit card companies and large computer-equipped corporations. +The police strongly urge these people, as good citizens, +to report offenses and press criminal charges; +they pass the message that there is someone in authority who cares, +understands, and, best of all, will take useful action +should a computer-crime occur. + +But reassuring talk is cheap. Sundevil offered action. + +The final message of Sundevil was intended for internal consumption +by law enforcement. Sundevil was offered as proof that the community +of American computer-crime police had come of age. Sundevil was +proof that enormous things like Sundevil itself could now be accomplished. +Sundevil was proof that the Secret Service and its local law-enforcement +allies could act like a well-oiled machine--(despite the hampering use +of those scrambled phones). It was also proof that the Arizona Organized +Crime and Racketeering Unit--the sparkplug of Sundevil--ranked with the best +in the world in ambition, organization, and sheer conceptual daring. + +And, as a final fillip, Sundevil was a message from the Secret Service +to their longtime rivals in the Federal Bureau of Investigation. +By Congressional fiat, both USSS and FBI formally share jurisdiction +over federal computer-crimebusting activities. Neither of these groups +has ever been remotely happy with this muddled situation. It seems to +suggest that Congress cannot make up its mind as to which of these groups +is better qualified. And there is scarcely a G-man or a Special Agent +anywhere without a very firm opinion on that topic. + +# + +For the neophyte, one of the most puzzling aspects of the crackdown +on hackers is why the United States Secret Service has anything at all +to do with this matter. + +The Secret Service is best known for its primary public role: +its agents protect the President of the United States. +They also guard the President's family, the Vice President and his family, +former Presidents, and Presidential candidates. They sometimes guard +foreign dignitaries who are visiting the United States, especially foreign +heads of state, and have been known to accompany American officials +on diplomatic missions overseas. + +Special Agents of the Secret Service don't wear uniforms, but the +Secret Service also has two uniformed police agencies. There's the +former White House Police (now known as the Secret Service Uniformed Division, +since they currently guard foreign embassies in Washington, as well as the +White House itself). And there's the uniformed Treasury Police Force. + +The Secret Service has been charged by Congress with a number +of little-known duties. They guard the precious metals in Treasury vaults. +They guard the most valuable historical documents of the United States: +originals of the Constitution, the Declaration of Independence, +Lincoln's Second Inaugural Address, an American-owned copy of +the Magna Carta, and so forth. Once they were assigned to guard +the Mona Lisa, on her American tour in the 1960s. + +The entire Secret Service is a division of the Treasury Department. +Secret Service Special Agents (there are about 1,900 of them) +are bodyguards for the President et al, but they all work for the Treasury. +And the Treasury (through its divisions of the U.S. Mint and the +Bureau of Engraving and Printing) prints the nation's money. + +As Treasury police, the Secret Service guards the nation's currency; +it is the only federal law enforcement agency with direct jurisdiction +over counterfeiting and forgery. It analyzes documents for authenticity, +and its fight against fake cash is still quite lively (especially since +the skilled counterfeiters of Medellin, Columbia have gotten into the act). +Government checks, bonds, and other obligations, which exist in untold +millions and are worth untold billions, are common targets for forgery, +which the Secret Service also battles. It even handles forgery +of postage stamps. + +But cash is fading in importance today as money has become electronic. +As necessity beckoned, the Secret Service moved from fighting the +counterfeiting of paper currency and the forging of checks, +to the protection of funds transferred by wire. + +From wire-fraud, it was a simple skip-and-jump to what is formally +known as "access device fraud." Congress granted the Secret Service +the authority to investigate "access device fraud" under Title 18 +of the United States Code (U.S.C. Section 1029). + +The term "access device" seems intuitively simple. It's some kind +of high-tech gizmo you use to get money with. It makes good sense +to put this sort of thing in the charge of counterfeiting and +wire-fraud experts. + +However, in Section 1029, the term "access device" is very +generously defined. An access device is: "any card, plate, +code, account number, or other means of account access +that can be used, alone or in conjunction with another access device, +to obtain money, goods, services, or any other thing of value, +or that can be used to initiate a transfer of funds." + +"Access device" can therefore be construed to include credit cards +themselves (a popular forgery item nowadays). It also includes credit card +account NUMBERS, those standards of the digital underground. The same goes +for telephone charge cards (an increasingly popular item with telcos, +who are tired of being robbed of pocket change by phone-booth thieves). +And also telephone access CODES, those OTHER standards of the digital +underground. (Stolen telephone codes may not "obtain money," but they +certainly do obtain valuable "services," which is specifically forbidden +by Section 1029.) + +We can now see that Section 1029 already pits the United States Secret Service +directly against the digital underground, without any mention at all of +the word "computer." + +Standard phreaking devices, like "blue boxes," used to steal phone service +from old-fashioned mechanical switches, are unquestionably "counterfeit +access devices." Thanks to Sec.1029, it is not only illegal to USE +counterfeit access devices, but it is even illegal to BUILD them. +"Producing," "designing" "duplicating" or "assembling" blue boxes +are all federal crimes today, and if you do this, the Secret Service +has been charged by Congress to come after you. + +Automatic Teller Machines, which replicated all over America during the 1980s, +are definitely "access devices," too, and an attempt to tamper with their +punch-in codes and plastic bank cards falls directly under Sec. 1029. + +Section 1029 is remarkably elastic. Suppose you find a computer password +in somebody's trash. That password might be a "code"--it's certainly a +"means of account access." Now suppose you log on to a computer +and copy some software for yourself. You've certainly obtained +"service" (computer service) and a "thing of value" (the software). +Suppose you tell a dozen friends about your swiped password, +and let them use it, too. Now you're "trafficking in unauthorized +access devices." And when the Prophet, a member of the Legion of Doom, +passed a stolen telephone company document to Knight Lightning +at Phrack magazine, they were both charged under Sec. 1029! + +There are two limitations on Section 1029. First, the offense must +"affect interstate or foreign commerce" in order to become a matter +of federal jurisdiction. The term "affecting commerce" is not well defined; +but you may take it as a given that the Secret Service can take an interest +if you've done most anything that happens to cross a state line. +State and local police can be touchy about their jurisdictions, +and can sometimes be mulish when the feds show up. But when it comes +to computer-crime, the local police are pathetically grateful +for federal help--in fact they complain that they can't get enough of it. +If you're stealing long-distance service, you're almost certainly crossing +state lines, and you're definitely "affecting the interstate commerce" +of the telcos. And if you're abusing credit cards by ordering stuff +out of glossy catalogs from, say, Vermont, you're in for it. + +The second limitation is money. As a rule, the feds don't pursue +penny-ante offenders. Federal judges will dismiss cases that appear +to waste their time. Federal crimes must be serious; Section 1029 +specifies a minimum loss of a thousand dollars. + +We now come to the very next section of Title 18, which is Section 1030, +"Fraud and related activity in connection with computers." This statute +gives the Secret Service direct jurisdiction over acts of computer intrusion. +On the face of it, the Secret Service would now seem to command the field. +Section 1030, however, is nowhere near so ductile as Section 1029. + +The first annoyance is Section 1030(d), which reads: + +"(d) The United States Secret Service shall, +IN ADDITION TO ANY OTHER AGENCY HAVING SUCH AUTHORITY, +have the authority to investigate offenses under this section. +Such authority of the United States Secret Service shall be +exercised in accordance with an agreement which shall be entered +into by the Secretary of the Treasury AND THE ATTORNEY GENERAL." +(Author's italics.) [Represented by capitals.] + +The Secretary of the Treasury is the titular head of the Secret Service, +while the Attorney General is in charge of the FBI. In Section (d), +Congress shrugged off responsibility for the computer-crime turf-battle +between the Service and the Bureau, and made them fight it out all +by themselves. The result was a rather dire one for the Secret Service, +for the FBI ended up with exclusive jurisdiction over computer break-ins +having to do with national security, foreign espionage, federally insured +banks, and U.S. military bases, while retaining joint jurisdiction over +all the other computer intrusions. Essentially, when it comes to Section 1030, +the FBI not only gets the real glamor stuff for itself, but can peer over the +shoulder of the Secret Service and barge in to meddle whenever it suits them. + +The second problem has to do with the dicey term +"Federal interest computer." Section 1030(a)(2) +makes it illegal to "access a computer without authorization" +if that computer belongs to a financial institution or an issuer +of credit cards (fraud cases, in other words). Congress was quite +willing to give the Secret Service jurisdiction over +money-transferring computers, but Congress balked at +letting them investigate any and all computer intrusions. +Instead, the USSS had to settle for the money machines +and the "Federal interest computers." A "Federal interest computer" +is a computer which the government itself owns, or is using. +Large networks of interstate computers, linked over state lines, +are also considered to be of "Federal interest." (This notion of +"Federal interest" is legally rather foggy and has never been +clearly defined in the courts. The Secret Service has never yet +had its hand slapped for investigating computer break-ins that were NOT +of "Federal interest," but conceivably someday this might happen.) + +So the Secret Service's authority over "unauthorized access" +to computers covers a lot of territory, but by no means the +whole ball of cyberspatial wax. If you are, for instance, +a LOCAL computer retailer, or the owner of a LOCAL bulletin +board system, then a malicious LOCAL intruder can break in, +crash your system, trash your files and scatter viruses, +and the U.S. Secret Service cannot do a single thing about it. + +At least, it can't do anything DIRECTLY. But the Secret Service +will do plenty to help the local people who can. + +The FBI may have dealt itself an ace off the bottom of the deck +when it comes to Section 1030; but that's not the whole story; +that's not the street. What's Congress thinks is one thing, +and Congress has been known to change its mind. The REAL +turf-struggle is out there in the streets where it's happening. +If you're a local street-cop with a computer problem, +the Secret Service wants you to know where you can find +the real expertise. While the Bureau crowd are off having +their favorite shoes polished--(wing-tips)--and making derisive +fun of the Service's favorite shoes--("pansy-ass tassels")-- +the tassel-toting Secret Service has a crew of ready-and-able +hacker-trackers installed in the capital of every state in the Union. +Need advice? They'll give you advice, or at least point you in +the right direction. Need training? They can see to that, too. + +If you're a local cop and you call in the FBI, the FBI +(as is widely and slanderously rumored) will order you around +like a coolie, take all the credit for your busts, +and mop up every possible scrap of reflected glory. +The Secret Service, on the other hand, doesn't brag a lot. +They're the quiet types. VERY quiet. Very cool. Efficient. +High-tech. Mirrorshades, icy stares, radio ear-plugs, +an Uzi machine-pistol tucked somewhere in that well-cut jacket. +American samurai, sworn to give their lives to protect our President. +"The granite agents." Trained in martial arts, absolutely fearless. +Every single one of 'em has a top-secret security clearance. +Something goes a little wrong, you're not gonna hear any whining +and moaning and political buck-passing out of these guys. + +The facade of the granite agent is not, of course, the reality. +Secret Service agents are human beings. And the real glory +in Service work is not in battling computer crime--not yet, +anyway--but in protecting the President. The real glamour +of Secret Service work is in the White House Detail. +If you're at the President's side, then the kids and the wife +see you on television; you rub shoulders with the most powerful +people in the world. That's the real heart of Service work, +the number one priority. More than one computer investigation +has stopped dead in the water when Service agents vanished at +the President's need. + +There's romance in the work of the Service. The intimate access +to circles of great power; the esprit-de-corps of a highly trained +and disciplined elite; the high responsibility of defending the +Chief Executive; the fulfillment of a patriotic duty. And as police +work goes, the pay's not bad. But there's squalor in Service work, too. +You may get spat upon by protesters howling abuse--and if they get violent, +if they get too close, sometimes you have to knock one of them down-- +discreetly. + +The real squalor in Service work is drudgery such as "the quarterlies," +traipsing out four times a year, year in, year out, to interview the various +pathetic wretches, many of them in prisons and asylums, who have seen fit +to threaten the President's life. And then there's the grinding stress +of searching all those faces in the endless bustling crowds, looking for +hatred, looking for psychosis, looking for the tight, nervous face +of an Arthur Bremer, a Squeaky Fromme, a Lee Harvey Oswald. +It's watching all those grasping, waving hands for sudden movements, +while your ears strain at your radio headphone for the long-rehearsed +cry of "Gun!" + +It's poring, in grinding detail, over the biographies of every rotten +loser who ever shot at a President. It's the unsung work of the +Protective Research Section, who study scrawled, anonymous death threats +with all the meticulous tools of anti-forgery techniques. + +And it's maintaining the hefty computerized files on anyone +who ever threatened the President's life. Civil libertarians +have become increasingly concerned at the Government's use +of computer files to track American citizens--but the +Secret Service file of potential Presidential assassins, +which has upward of twenty thousand names, rarely causes +a peep of protest. If you EVER state that you intend to +kill the President, the Secret Service will want to know +and record who you are, where you are, what you are, +and what you're up to. If you're a serious threat-- +if you're officially considered "of protective interest"-- +then the Secret Service may well keep tabs on you +for the rest of your natural life. + +Protecting the President has first call on all the Service's resources. +But there's a lot more to the Service's traditions and history than +standing guard outside the Oval Office. + +The Secret Service is the nation's oldest general federal +law-enforcement agency. Compared to the Secret Service, +the FBI are new-hires and the CIA are temps. The Secret Service +was founded 'way back in 1865, at the suggestion of Hugh McCulloch, +Abraham Lincoln's Secretary of the Treasury. McCulloch wanted +a specialized Treasury police to combat counterfeiting. +Abraham Lincoln agreed that this seemed a good idea, and, +with a terrible irony, Abraham Lincoln was shot that +very night by John Wilkes Booth. + +The Secret Service originally had nothing to do with protecting Presidents. +They didn't take this on as a regular assignment until after the Garfield +assassination in 1881. And they didn't get any Congressional money for it +until President McKinley was shot in 1901. The Service was originally +designed for one purpose: destroying counterfeiters. + +# + +There are interesting parallels between the Service's +nineteenth-century entry into counterfeiting, +and America's twentieth-century entry into computer-crime. + +In 1865, America's paper currency was a terrible muddle. +Security was drastically bad. Currency was printed on the spot +by local banks in literally hundreds of different designs. +No one really knew what the heck a dollar bill was supposed to look like. +Bogus bills passed easily. If some joker told you that a one-dollar bill +from the Railroad Bank of Lowell, Massachusetts had a woman leaning on +a shield, with a locomotive, a cornucopia, a compass, various agricultural +implements, a railroad bridge, and some factories, then you pretty much had +to take his word for it. (And in fact he was telling the truth!) + +SIXTEEN HUNDRED local American banks designed and printed their own +paper currency, and there were no general standards for security. +Like a badly guarded node in a computer network, badly designed bills +were easy to fake, and posed a security hazard for the entire monetary system. + +No one knew the exact extent of the threat to the currency. +There were panicked estimates that as much as a third of +the entire national currency was faked. Counterfeiters-- +known as "boodlers" in the underground slang of the time-- +were mostly technically skilled printers who had gone to the bad. +Many had once worked printing legitimate currency. +Boodlers operated in rings and gangs. Technical experts +engraved the bogus plates--commonly in basements in New York City. +Smooth confidence men passed large wads of high-quality, +high-denomination fakes, including the really sophisticated stuff-- +government bonds, stock certificates, and railway shares. +Cheaper, botched fakes were sold or sharewared to low-level +gangs of boodler wannabes. (The really cheesy lowlife boodlers +merely upgraded real bills by altering face values, +changing ones to fives, tens to hundreds, and so on.) + +The techniques of boodling were little-known and regarded +with a certain awe by the mid- nineteenth-century public. +The ability to manipulate the system for rip-off seemed +diabolically clever. As the skill and daring of the +boodlers increased, the situation became intolerable. +The federal government stepped in, and began offering +its own federal currency, which was printed in fancy green ink, +but only on the back--the original "greenbacks." And at first, +the improved security of the well-designed, well-printed +federal greenbacks seemed to solve the problem; but then +the counterfeiters caught on. Within a few years things were +worse than ever: a CENTRALIZED system where ALL security was bad! + +The local police were helpless. The Government tried offering +blood money to potential informants, but this met with little success. +Banks, plagued by boodling, gave up hope of police help and hired +private security men instead. Merchants and bankers queued up +by the thousands to buy privately-printed manuals on currency security, +slim little books like Laban Heath's INFALLIBLE GOVERNMENT +COUNTERFEIT DETECTOR. The back of the book offered Laban Heath's +patent microscope for five bucks. + +Then the Secret Service entered the picture. The first agents +were a rough and ready crew. Their chief was one William P. Wood, +a former guerilla in the Mexican War who'd won a reputation busting +contractor fraudsters for the War Department during the Civil War. +Wood, who was also Keeper of the Capital Prison, had a sideline +as a counterfeiting expert, bagging boodlers for the federal bounty money. + +Wood was named Chief of the new Secret Service in July 1865. +There were only ten Secret Service agents in all: Wood himself, +a handful who'd worked for him in the War Department, and a few +former private investigators--counterfeiting experts--whom Wood +had won over to public service. (The Secret Service of 1865 was +much the size of the Chicago Computer Fraud Task Force or the +Arizona Racketeering Unit of 1990.) These ten "Operatives" +had an additional twenty or so "Assistant Operatives" and "Informants." +Besides salary and per diem, each Secret Service employee received +a whopping twenty-five dollars for each boodler he captured. + +Wood himself publicly estimated that at least HALF of America's currency +was counterfeit, a perhaps pardonable perception. Within a year the +Secret Service had arrested over 200 counterfeiters. They busted about +two hundred boodlers a year for four years straight. + +Wood attributed his success to travelling fast and light, hitting the +bad-guys hard, and avoiding bureaucratic baggage. "Because my raids +were made without military escort and I did not ask the assistance +of state officers, I surprised the professional counterfeiter." + +Wood's social message to the once-impudent boodlers bore an eerie ring +of Sundevil: "It was also my purpose to convince such characters that +it would no longer be healthy for them to ply their vocation without +being handled roughly, a fact they soon discovered." + +William P. Wood, the Secret Service's guerilla pioneer, +did not end well. He succumbed to the lure of aiming for +the really big score. The notorious Brockway Gang of New York City, +headed by William E. Brockway, the "King of the Counterfeiters," +had forged a number of government bonds. They'd passed these +brilliant fakes on the prestigious Wall Street investment +firm of Jay Cooke and Company. The Cooke firm were frantic +and offered a huge reward for the forgers' plates. + +Laboring diligently, Wood confiscated the plates +(though not Mr. Brockway) and claimed the reward. +But the Cooke company treacherously reneged. +Wood got involved in a down-and-dirty lawsuit +with the Cooke capitalists. Wood's boss, +Secretary of the Treasury McCulloch, felt that +Wood's demands for money and glory were unseemly, +and even when the reward money finally came through, +McCulloch refused to pay Wood anything. +Wood found himself mired in a seemingly endless +round of federal suits and Congressional lobbying. + +Wood never got his money. And he lost his job to boot. +He resigned in 1869. + +Wood's agents suffered, too. On May 12, 1869, the second Chief +of the Secret Service took over, and almost immediately fired +most of Wood's pioneer Secret Service agents: Operatives, +Assistants and Informants alike. The practice of receiving $25 +per crook was abolished. And the Secret Service began the long, +uncertain process of thorough professionalization. + +Wood ended badly. He must have felt stabbed in the back. +In fact his entire organization was mangled. + +On the other hand, William P. Wood WAS the first head of the Secret Service. +William Wood was the pioneer. People still honor his name. Who remembers +the name of the SECOND head of the Secret Service? + +As for William Brockway (also known as "Colonel Spencer"), +he was finally arrested by the Secret Service in 1880. +He did five years in prison, got out, and was still boodling +at the age of seventy-four. + +# + +Anyone with an interest in Operation Sundevil-- +or in American computer-crime generally-- +could scarcely miss the presence of Gail Thackeray, +Assistant Attorney General of the State of Arizona. +Computer-crime training manuals often cited +Thackeray's group and her work; she was the +highest-ranking state official to specialize +in computer-related offenses. Her name had been +on the Sundevil press release (though modestly ranked +well after the local federal prosecuting attorney and +the head of the Phoenix Secret Service office). + +As public commentary, and controversy, began to mount +about the Hacker Crackdown, this Arizonan state official +began to take a higher and higher public profile. +Though uttering almost nothing specific about +the Sundevil operation itself, she coined some +of the most striking soundbites of the growing propaganda war: +"Agents are operating in good faith, and I don't think +you can say that for the hacker community," was one. +Another was the memorable "I am not a mad dog prosecutor" +(Houston Chronicle, Sept 2, 1990.) In the meantime, +the Secret Service maintained its usual extreme discretion; +the Chicago Unit, smarting from the backlash +of the Steve Jackson scandal, had gone completely to earth. + +As I collated my growing pile of newspaper clippings, +Gail Thackeray ranked as a comparative fount of public +knowledge on police operations. + +I decided that I had to get to know Gail Thackeray. +I wrote to her at the Arizona Attorney General's Office. +Not only did she kindly reply to me, but, to my astonishment, +she knew very well what "cyberpunk" science fiction was. + +Shortly after this, Gail Thackeray lost her job. +And I temporarily misplaced my own career as +a science-fiction writer, to become a full-time +computer-crime journalist. In early March, 1991, +I flew to Phoenix, Arizona, to interview Gail Thackeray +for my book on the hacker crackdown. + +# + +"Credit cards didn't used to cost anything to get," +says Gail Thackeray. "Now they cost forty bucks-- +and that's all just to cover the costs from RIP-OFF ARTISTS." + +Electronic nuisance criminals are parasites. +One by one they're not much harm, no big deal. +But they never come just one by one. They come in swarms, +heaps, legions, sometimes whole subcultures. And they bite. +Every time we buy a credit card today, we lose a little financial +vitality to a particular species of bloodsucker. + +What, in her expert opinion, are the worst forms of electronic crime, +I ask, consulting my notes. Is it--credit card fraud? Breaking into +ATM bank machines? Phone-phreaking? Computer intrusions? +Software viruses? Access-code theft? Records tampering? +Software piracy? Pornographic bulletin boards? +Satellite TV piracy? Theft of cable service? +It's a long list. By the time I reach the end +of it I feel rather depressed. + +"Oh no," says Gail Thackeray, leaning forward over the table, +her whole body gone stiff with energetic indignation, +"the biggest damage is telephone fraud. Fake sweepstakes, +fake charities. Boiler-room con operations. You could pay off +the national debt with what these guys steal. . . . +They target old people, they get hold of credit ratings +and demographics, they rip off the old and the weak." +The words come tumbling out of her. + +It's low-tech stuff, your everyday boiler-room fraud. +Grifters, conning people out of money over the phone, +have been around for decades. This is where the word "phony" came from! + +It's just that it's so much EASIER now, horribly facilitated by advances +in technology and the byzantine structure of the modern phone system. +The same professional fraudsters do it over and over, Thackeray tells me, +they hide behind dense onion-shells of fake companies. . . fake holding +corporations nine or ten layers deep, registered all over the map. +They get a phone installed under a false name in an empty safe-house. +And then they call-forward everything out of that phone to yet +another phone, a phone that may even be in another STATE. +And they don't even pay the charges on their phones; +after a month or so, they just split; set up somewhere else +in another Podunkville with the same seedy crew of veteran phone-crooks. +They buy or steal commercial credit card reports, slap them on the PC, +have a program pick out people over sixty-five who pay a lot to charities. +A whole subculture living off this, merciless folks on the con. + +"The `light-bulbs for the blind' people," Thackeray muses, +with a special loathing. "There's just no end to them." + +We're sitting in a downtown diner in Phoenix, Arizona. +It's a tough town, Phoenix. A state capital seeing some hard times. +Even to a Texan like myself, Arizona state politics seem rather baroque. +There was, and remains, endless trouble over the Martin Luther King holiday, +the sort of stiff-necked, foot-shooting incident for which Arizona politics +seem famous. There was Evan Mecham, the eccentric Republican millionaire +governor who was impeached, after reducing state government to a +ludicrous shambles. Then there was the national Keating scandal, +involving Arizona savings and loans, in which both of Arizona's +U.S. senators, DeConcini and McCain, played sadly prominent roles. + +And the very latest is the bizarre AzScam case, +in which state legislators were videotaped, +eagerly taking cash from an informant of the Phoenix city +police department, who was posing as a Vegas mobster. + +"Oh," says Thackeray cheerfully. "These people are amateurs here, +they thought they were finally getting to play with the big boys. +They don't have the least idea how to take a bribe! +It's not institutional corruption. It's not like back in Philly." + +Gail Thackeray was a former prosecutor in Philadelphia. +Now she's a former assistant attorney general of the State of Arizona. +Since moving to Arizona in 1986, she had worked under the aegis +of Steve Twist, her boss in the Attorney General's office. +Steve Twist wrote Arizona's pioneering computer crime laws +and naturally took an interest in seeing them enforced. +It was a snug niche, and Thackeray's Organized Crime and +Racketeering Unit won a national reputation for ambition +and technical knowledgeability. . . . Until the latest +election in Arizona. Thackeray's boss ran for the top +job, and lost. The victor, the new Attorney General, +apparently went to some pains to eliminate the bureaucratic +traces of his rival, including his pet group--Thackeray's group. +Twelve people got their walking papers. + +Now Thackeray's painstakingly assembled computer lab +sits gathering dust somewhere in the glass-and-concrete +Attorney General's HQ on 1275 Washington Street. +Her computer-crime books, her painstakingly garnered +back issues of phreak and hacker zines, all bought +at her own expense--are piled in boxes somewhere. +The State of Arizona is simply not particularly +interested in electronic racketeering at the moment. + +At the moment of our interview, Gail Thackeray, +officially unemployed, is working out of the county +sheriff's office, living on her savings, and prosecuting +several cases--working 60-hour weeks, just as always-- +for no pay at all. "I'm trying to train people," +she mutters. + +Half her life seems to be spent training people--merely pointing out, +to the naive and incredulous (such as myself) that this stuff +is ACTUALLY GOING ON OUT THERE. It's a small world, computer crime. +A young world. Gail Thackeray, a trim blonde Baby-Boomer who favors +Grand Canyon white-water rafting to kill some slow time, +is one of the world's most senior, most veteran "hacker-trackers." +Her mentor was Donn Parker, the California think-tank theorist +who got it all started `way back in the mid-70s, the "grandfather +of the field," "the great bald eagle of computer crime." + +And what she has learned, Gail Thackeray teaches. Endlessly. +Tirelessly. To anybody. To Secret Service agents and state police, +at the Glynco, Georgia federal training center. To local police, +on "roadshows" with her slide projector and notebook. +To corporate security personnel. To journalists. To parents. + +Even CROOKS look to Gail Thackeray for advice. +Phone-phreaks call her at the office. They know very +well who she is. They pump her for information +on what the cops are up to, how much they know. +Sometimes whole CROWDS of phone phreaks, +hanging out on illegal conference calls, will call Gail +Thackeray up. They taunt her. And, as always, +they boast. Phone-phreaks, real stone phone-phreaks, +simply CANNOT SHUT UP. They natter on for hours. + +Left to themselves, they mostly talk about the intricacies +of ripping-off phones; it's about as interesting as listening +to hot-rodders talk about suspension and distributor-caps. +They also gossip cruelly about each other. And when talking +to Gail Thackeray, they incriminate themselves. "I have tapes," +Thackeray says coolly. + +Phone phreaks just talk like crazy. "Dial-Tone" out in Alabama +has been known to spend half-an-hour simply reading stolen +phone-codes aloud into voice-mail answering machines. +Hundreds, thousands of numbers, recited in a monotone, +without a break--an eerie phenomenon. When arrested, +it's a rare phone phreak who doesn't inform at endless length +on everybody he knows. + +Hackers are no better. What other group of criminals, +she asks rhetorically, publishes newsletters and holds conventions? +She seems deeply nettled by the sheer brazenness of this behavior, +though to an outsider, this activity might make one wonder +whether hackers should be considered "criminals" at all. +Skateboarders have magazines, and they trespass a lot. +Hot rod people have magazines and they break speed limits +and sometimes kill people. . . . + +I ask her whether it would be any loss to society if phone phreaking +and computer hacking, as hobbies, simply dried up and blew away, +so that nobody ever did it again. + +She seems surprised. "No," she says swiftly. "Maybe a little. . . +in the old days. . .the MIT stuff. . . . But there's a lot of wonderful, +legal stuff you can do with computers now, you don't have to break into +somebody else's just to learn. You don't have that excuse. +You can learn all you like." + +Did you ever hack into a system? I ask. + +The trainees do it at Glynco. Just to demonstrate system vulnerabilities. +She's cool to the notion. Genuinely indifferent. + +"What kind of computer do you have?" + +"A Compaq 286LE," she mutters. + +"What kind do you WISH you had?" + +At this question, the unmistakable light of true hackerdom flares in +Gail Thackeray's eyes. She becomes tense, animated, the words pour out: +"An Amiga 2000 with an IBM card and Mac emulation! The most common hacker +machines are Amigas and Commodores. And Apples." If she had the Amiga, +she enthuses, she could run a whole galaxy of seized computer-evidence disks +on one convenient multifunctional machine. A cheap one, too. Not like the +old Attorney General lab, where they had an ancient CP/M machine, +assorted Amiga flavors and Apple flavors, a couple IBMS, all the +utility software. . .but no Commodores. The workstations down +at the Attorney General's are Wang dedicated word-processors. +Lame machines tied in to an office net--though at least they get +on- line to the Lexis and Westlaw legal data services. + +I don't say anything. I recognize the syndrome, though. +This computer-fever has been running through segments of +our society for years now. It's a strange kind of lust: +K-hunger, Meg-hunger; but it's a shared disease; +it can kill parties dead, as conversation spirals into +the deepest and most deviant recesses of software releases +and expensive peripherals. . . . The mark of the hacker beast. +I have it too. The whole "electronic community," whatever the hell +that is, has it. Gail Thackeray has it. Gail Thackeray is a hacker cop. +My immediate reaction is a strong rush of indignant pity: +WHY DOESN'T SOMEBODY BUY THIS WOMAN HER AMIGA?! +It's not like she's asking for a Cray X-MP +supercomputer mainframe; an Amiga's a sweet little +cookie-box thing. We're losing zillions in organized fraud; +prosecuting and defending a single hacker case in court can cost +a hundred grand easy. How come nobody can come up with four lousy grand +so this woman can do her job? For a hundred grand we could buy every +computer cop in America an Amiga. There aren't that many of 'em. + +Computers. The lust, the hunger, for computers. +The loyalty they inspire, the intense sense of possessiveness. +The culture they have bred. I myself am sitting in downtown Phoenix, +Arizona because it suddenly occurred to me that the police might-- +just MIGHT--come and take away my computer. The prospect of this, +the mere IMPLIED THREAT, was unbearable. It literally changed my life. +It was changing the lives of many others. Eventually it would change +everybody's life. + +Gail Thackeray was one of the top computer-crime people in America. +And I was just some novelist, and yet I had a better computer than hers. +PRACTICALLY EVERYBODY I KNEW had a better computer than Gail Thackeray +and her feeble laptop 286. It was like sending the sheriff in to clean +up Dodge City and arming her with a slingshot cut from an old rubber tire. + +But then again, you don't need a howitzer to enforce the law. +You can do a lot just with a badge. With a badge alone, +you can basically wreak havoc, take a terrible vengeance on wrongdoers. +Ninety percent of "computer crime investigation" is just "crime investigation:" +names, places, dossiers, modus operandi, search warrants, victims, +complainants, informants. . . . + +What will computer crime look like in ten years? Will it get better? +Did "Sundevil" send 'em reeling back in confusion? + +It'll be like it is now, only worse, she tells me with perfect conviction. +Still there in the background, ticking along, changing with the times: +the criminal underworld. It'll be like drugs are. Like our problems +with alcohol. All the cops and laws in the world never solved our problems +with alcohol. If there's something people want, a certain percentage +of them are just going to take it. Fifteen percent of the populace +will never steal. Fifteen percent will steal most anything not nailed down. +The battle is for the hearts and minds of the remaining seventy percent. + +And criminals catch on fast. If there's not "too steep a learning curve"-- +if it doesn't require a baffling amount of expertise and practice-- +then criminals are often some of the first through the gate of a +new technology. Especially if it helps them to hide. +They have tons of cash, criminals. The new communications tech-- +like pagers, cellular phones, faxes, Federal Express--were pioneered +by rich corporate people, and by criminals. In the early years +of pagers and beepers, dope dealers were so enthralled this technology +that owing a beeper was practically prima facie evidence of cocaine dealing. +CB radio exploded when the speed limit hit 55 and breaking the highway law +became a national pastime. Dope dealers send cash by Federal Express, +despite, or perhaps BECAUSE OF, the warnings in FedEx offices that tell you +never to try this. Fed Ex uses X-rays and dogs on their mail, +to stop drug shipments. That doesn't work very well. + +Drug dealers went wild over cellular phones. +There are simple methods of faking ID on cellular phones, +making the location of the call mobile, free of charge, +and effectively untraceable. Now victimized cellular +companies routinely bring in vast toll-lists of calls +to Colombia and Pakistan. + +Judge Greene's fragmentation of the phone company +is driving law enforcement nuts. Four thousand +telecommunications companies. Fraud skyrocketing. +Every temptation in the world available with a phone +and a credit card number. Criminals untraceable. +A galaxy of "new neat rotten things to do." + +If there were one thing Thackeray would like to have, +it would be an effective legal end-run through this new +fragmentation minefield. + +It would be a new form of electronic search warrant, +an "electronic letter of marque" to be issued by a judge. +It would create a new category of "electronic emergency." +Like a wiretap, its use would be rare, but it would cut +across state lines and force swift cooperation from all concerned. +Cellular, phone, laser, computer network, PBXes, AT&T, Baby Bells, +long-distance entrepreneurs, packet radio. Some document, +some mighty court-order, that could slice through four thousand +separate forms of corporate red-tape, and get her at once to +the source of calls, the source of email threats and viruses, +the sources of bomb threats, kidnapping threats. "From now on," +she says, "the Lindbergh baby will always die." + +Something that would make the Net sit still, if only for a moment. +Something that would get her up to speed. Seven league boots. +That's what she really needs. "Those guys move in nanoseconds +and I'm on the Pony Express." + +And then, too, there's the coming international angle. +Electronic crime has never been easy to localize, +to tie to a physical jurisdiction. And phone-phreaks +and hackers loathe boundaries, they jump them whenever they can. +The English. The Dutch. And the Germans, especially the ubiquitous +Chaos Computer Club. The Australians. They've all learned phone-phreaking +from America. It's a growth mischief industry. The multinational +networks are global, but governments and the police simply aren't. +Neither are the laws. Or the legal frameworks for citizen protection. + +One language is global, though--English. Phone phreaks speak English; +it's their native tongue even if they're Germans. English may have started +in England but now it's the Net language; it might as well be called "CNNese." + +Asians just aren't much into phone phreaking. They're the world masters +at organized software piracy. The French aren't into phone-phreaking either. +The French are into computerized industrial espionage. + +In the old days of the MIT righteous hackerdom, crashing systems +didn't hurt anybody. Not all that much, anyway. Not permanently. +Now the players are more venal. Now the consequences are worse. +Hacking will begin killing people soon. Already there are methods +of stacking calls onto 911 systems, annoying the police, and possibly +causing the death of some poor soul calling in with a genuine emergency. +Hackers in Amtrak computers, or air-traffic control computers, will kill +somebody someday. Maybe a lot of people. Gail Thackeray expects it. + +And the viruses are getting nastier. The "Scud" virus is the latest one out. +It wipes hard-disks. + +According to Thackeray, the idea that phone-phreaks are Robin Hoods is a fraud. +They don't deserve this repute. Basically, they pick on the weak. AT&T now +protects itself with the fearsome ANI (Automatic Number Identification) +trace capability. When AT&T wised up and tightened security generally, +the phreaks drifted into the Baby Bells. The Baby Bells lashed out in 1989 +and 1990, so the phreaks switched to smaller long-distance entrepreneurs. +Today, they are moving into locally owned PBXes and voice-mail systems, +which are full of security holes, dreadfully easy to hack. These victims +aren't the moneybags Sheriff of Nottingham or Bad King John, but small groups +of innocent people who find it hard to protect themselves, and who really +suffer from these depredations. Phone phreaks pick on the weak. They do it +for power. If it were legal, they wouldn't do it. They don't want service, +or knowledge, they want the thrill of power-tripping. There's plenty of +knowledge or service around if you're willing to pay. Phone phreaks don't pay, +they steal. It's because it is illegal that it feels like power, +that it gratifies their vanity. + +I leave Gail Thackeray with a handshake at the door of her office building-- +a vast International-Style office building downtown. The Sheriff's office +is renting part of it. I get the vague impression that quite a lot of the +building is empty--real estate crash. + +In a Phoenix sports apparel store, in a downtown mall, I meet +the "Sun Devil" himself. He is the cartoon mascot of +Arizona State University, whose football stadium, "Sundevil," +is near the local Secret Service HQ--hence the name Operation Sundevil. +The Sun Devil himself is named "Sparky." Sparky the Sun Devil is maroon +and bright yellow, the school colors. Sparky brandishes a three-tined +yellow pitchfork. He has a small mustache, pointed ears, a barbed tail, +and is dashing forward jabbing the air with the pitchfork, +with an expression of devilish glee. + +Phoenix was the home of Operation Sundevil. The Legion of Doom +ran a hacker bulletin board called "The Phoenix Project." +An Australian hacker named "Phoenix" once burrowed through +the Internet to attack Cliff Stoll, then bragged and boasted +about it to The New York Times. This net of coincidence +is both odd and meaningless. + +The headquarters of the Arizona Attorney General, Gail Thackeray's +former workplace, is on 1275 Washington Avenue. Many of the downtown +streets in Phoenix are named after prominent American presidents: +Washington, Jefferson, Madison. . . . + +After dark, all the employees go home to their suburbs. +Washington, Jefferson and Madison--what would be the +Phoenix inner city, if there were an inner city in this +sprawling automobile-bred town--become the haunts +of transients and derelicts. The homeless. The sidewalks +along Washington are lined with orange trees. +Ripe fallen fruit lies scattered like croquet balls +on the sidewalks and gutters. No one seems to be eating them. +I try a fresh one. It tastes unbearably bitter. + +The Attorney General's office, built in 1981 during the +Babbitt administration, is a long low two-story building +of white cement and wall-sized sheets of curtain-glass. +Behind each glass wall is a lawyer's office, quite open +and visible to anyone strolling by. Across the street +is a dour government building labelled simply ECONOMIC SECURITY, +something that has not been in great supply in the American +Southwest lately. + +The offices are about twelve feet square. They feature +tall wooden cases full of red-spined lawbooks; +Wang computer monitors; telephones; Post-it notes galore. +Also framed law diplomas and a general excess of bad +Western landscape art. Ansel Adams photos are a big favorite, +perhaps to compensate for the dismal specter of the parking lot, +two acres of striped black asphalt, which features gravel landscaping +and some sickly-looking barrel cacti. + +It has grown dark. Gail Thackeray has told me that the people +who work late here, are afraid of muggings in the parking lot. +It seems cruelly ironic that a woman tracing electronic racketeers +across the interstate labyrinth of Cyberspace should fear an assault +by a homeless derelict in the parking lot of her own workplace. + +Perhaps this is less than coincidence. Perhaps these two seemingly +disparate worlds are somehow generating one another. The poor and +disenfranchised take to the streets, while the rich and computer-equipped, +safe in their bedrooms, chatter over their modems. Quite often the derelicts +kick the glass out and break in to the lawyers' offices, if they see something +they need or want badly enough. + +I cross the parking lot to the street behind the Attorney General's office. +A pair of young tramps are bedding down on flattened sheets of cardboard, +under an alcove stretching over the sidewalk. One tramp wears a +glitter-covered T-shirt reading "CALIFORNIA" in Coca-Cola cursive. +His nose and cheeks look chafed and swollen; they glisten with +what seems to be Vaseline. The other tramp has a ragged long-sleeved +shirt and lank brown hair parted in the middle. They both wear blue jeans +coated in grime. They are both drunk. + +"You guys crash here a lot?" I ask them. + +They look at me warily. I am wearing black jeans, a black pinstriped +suit jacket and a black silk tie. I have odd shoes and a funny haircut. + +"It's our first time here," says the red-nosed tramp unconvincingly. +There is a lot of cardboard stacked here. More than any two people could use. + +"We usually stay at the Vinnie's down the street," says the brown-haired tramp, +puffing a Marlboro with a meditative air, as he sprawls with his head on +a blue nylon backpack. "The Saint Vincent's." + +"You know who works in that building over there?" I ask, pointing. + +The brown-haired tramp shrugs. "Some kind of attorneys, it says." + +We urge one another to take it easy. I give them five bucks. + +A block down the street I meet a vigorous workman who is wheeling along +some kind of industrial trolley; it has what appears to be a tank of +propane on it. + +We make eye contact. We nod politely. I walk past him. "Hey! +Excuse me sir!" he says. + +"Yes?" I say, stopping and turning. + +"Have you seen," the guy says rapidly, "a black guy, about 6'7", +scars on both his cheeks like this--" he gestures-- "wears a +black baseball cap on backwards, wandering around here anyplace?" + +"Sounds like I don't much WANT to meet him," I say. + +"He took my wallet," says my new acquaintance. +"Took it this morning. Y'know, some people would be +SCARED of a guy like that. But I'm not scared. +I'm from Chicago. I'm gonna hunt him down. +We do things like that in Chicago." + +"Yeah?" + +"I went to the cops and now he's got an APB out on his ass," +he says with satisfaction. "You run into him, you let me know." + +"Okay," I say. "What is your name, sir?" + +"Stanley. . . ." + +"And how can I reach you?" + +"Oh," Stanley says, in the same rapid voice, +"you don't have to reach, uh, me. +You can just call the cops. Go straight to the cops." +He reaches into a pocket and pulls out a greasy piece of pasteboard. +"See, here's my report on him." + +I look. The "report," the size of an index card, is labelled PRO-ACT: +Phoenix Residents Opposing Active Crime Threat. . . . or is it +Organized Against Crime Threat? In the darkening street it's hard +to read. Some kind of vigilante group? Neighborhood watch? +I feel very puzzled. + +"Are you a police officer, sir?" + +He smiles, seems very pleased by the question. + +"No," he says. + +"But you are a `Phoenix Resident?'" + +"Would you believe a homeless person," Stanley says. + +"Really? But what's with the. . . ." For the first time I take a close look +at Stanley's trolley. It's a rubber-wheeled thing of industrial metal, +but the device I had mistaken for a tank of propane is in fact a water-cooler. +Stanley also has an Army duffel-bag, stuffed tight as a sausage with clothing +or perhaps a tent, and, at the base of his trolley, a cardboard box and a +battered leather briefcase. + +"I see," I say, quite at a loss. For the first time I notice that Stanley +has a wallet. He has not lost his wallet at all. It is in his back pocket +and chained to his belt. It's not a new wallet. It seems to have seen +a lot of wear. + +"Well, you know how it is, brother," says Stanley. +Now that I know that he is homeless--A POSSIBLE +THREAT--my entire perception of him has changed +in an instant. His speech, which once seemed just +bright and enthusiastic, now seems to have a +dangerous tang of mania. "I have to do this!" +he assures me. "Track this guy down. . . . +It's a thing I do. . . you know. . .to keep myself together!" +He smiles, nods, lifts his trolley by its decaying rubber handgrips. + +"Gotta work together, y'know," Stanley booms, his face alight +with cheerfulness, "the police can't do everything!" +The gentlemen I met in my stroll in downtown Phoenix +are the only computer illiterates in this book. +To regard them as irrelevant, however, would be a grave mistake. + +As computerization spreads across society, the populace at large +is subjected to wave after wave of future shock. But, as a +necessary converse, the "computer community" itself is subjected +to wave after wave of incoming computer illiterates. +How will those currently enjoying America's digital bounty regard, +and treat, all this teeming refuse yearning to breathe free? +Will the electronic frontier be another Land of Opportunity-- +or an armed and monitored enclave, where the disenfranchised +snuggle on their cardboard at the locked doors of our houses of justice? + +Some people just don't get along with computers. They can't read. +They can't type. They just don't have it in their heads to master +arcane instructions in wirebound manuals. Somewhere, the process +of computerization of the populace will reach a limit. Some people-- +quite decent people maybe, who might have thrived in any other situation-- +will be left irretrievably outside the bounds. What's to be done with +these people, in the bright new shiny electroworld? How will they +be regarded, by the mouse-whizzing masters of cyberspace? With contempt? +Indifference? Fear? + +In retrospect, it astonishes me to realize how quickly poor Stanley +became a perceived threat. Surprise and fear are closely allied feelings. +And the world of computing is full of surprises. + +I met one character in the streets of Phoenix whose role in this book +is supremely and directly relevant. That personage was Stanley's giant +thieving scarred phantom. This phantasm is everywhere in this book. +He is the specter haunting cyberspace. + +Sometimes he's a maniac vandal ready to smash the phone system +for no sane reason at all. Sometimes he's a fascist fed, +coldly programming his mighty mainframes to destroy our Bill of Rights. +Sometimes he's a telco bureaucrat, covertly conspiring to register all modems +in the service of an Orwellian surveillance regime. Mostly, though, +this fearsome phantom is a "hacker." He's strange, he doesn't belong, +he's not authorized, he doesn't smell right, he's not keeping his proper place, +he's not one of us. The focus of fear is the hacker, for much the same +reasons that Stanley's fancied assailant is black. + +Stanley's demon can't go away, because he doesn't exist. +Despite singleminded and tremendous effort, he can't be arrested, +sued, jailed, or fired. The only constructive way to do ANYTHING +about him is to learn more about Stanley himself. This learning process +may be repellent, it may be ugly, it may involve grave elements of paranoiac +confusion, but it's necessary. Knowing Stanley requires something more +than class-crossing condescension. It requires more than steely +legal objectivity. It requires human compassion and sympathy. + +To know Stanley is to know his demon. If you know the other guy's demon, +then maybe you'll come to know some of your own. You'll be able to +separate reality from illusion. And then you won't do your cause, +and yourself, more harm than good. Like poor damned Stanley from Chicago did. + +# + +The Federal Computer Investigations Committee (FCIC) is the most important +and influential organization in the realm of American computer-crime. +Since the police of other countries have largely taken their computer-crime +cues from American methods, the FCIC might well be called the most important +computer crime group in the world. + +It is also, by federal standards, an organization of great unorthodoxy. +State and local investigators mix with federal agents. Lawyers, +financial auditors and computer-security programmers trade notes +with street cops. Industry vendors and telco security people show up +to explain their gadgetry and plead for protection and justice. +Private investigators, think-tank experts and industry pundits throw in +their two cents' worth. The FCIC is the antithesis of a formal bureaucracy. + +Members of the FCIC are obscurely proud of this fact; they recognize their +group as aberrant, but are entirely convinced that this, for them, +outright WEIRD behavior is nevertheless ABSOLUTELY NECESSARY +to get their jobs done. + +FCIC regulars --from the Secret Service, the FBI, the IRS, +the Department of Labor, the offices of federal attorneys, +state police, the Air Force, from military intelligence-- +often attend meetings, held hither and thither across the country, +at their own expense. The FCIC doesn't get grants. It doesn't +charge membership fees. It doesn't have a boss. It has no headquarters-- +just a mail drop in Washington DC, at the Fraud Division of the Secret Service. +It doesn't have a budget. It doesn't have schedules. It meets three times +a year--sort of. Sometimes it issues publications, but the FCIC +has no regular publisher, no treasurer, not even a secretary. +There are no minutes of FCIC meetings. Non-federal people are considered +"non-voting members," but there's not much in the way of elections. +There are no badges, lapel pins or certificates of membership. +Everyone is on a first-name basis. There are about forty of them. +Nobody knows how many, exactly. People come, people go-- +sometimes people "go" formally but still hang around anyway. +Nobody has ever exactly figured out what "membership" of this +"Committee" actually entails. + +Strange as this may seem to some, to anyone familiar with the social world +of computing, the "organization" of the FCIC is very recognizable. + +For years now, economists and management theorists have speculated +that the tidal wave of the information revolution would destroy rigid, +pyramidal bureaucracies, where everything is top-down and +centrally controlled. Highly trained "employees" would take on +much greater autonomy, being self-starting, and self-motivating, +moving from place to place, task to task, with great speed and fluidity. +"Ad-hocracy" would rule, with groups of people spontaneously knitting +together across organizational lines, tackling the problem at hand, +applying intense computer-aided expertise to it, and then vanishing +whence they came. + +This is more or less what has actually happened in the world of +federal computer investigation. With the conspicuous exception +of the phone companies, which are after all over a hundred years old, +practically EVERY organization that plays any important role in this book +functions just like the FCIC. The Chicago Task Force, the Arizona +Racketeering Unit, the Legion of Doom, the Phrack crowd, the +Electronic Frontier Foundation--they ALL look and act like "tiger teams" +or "user's groups." They are all electronic ad-hocracies leaping up +spontaneously to attempt to meet a need. + +Some are police. Some are, by strict definition, criminals. +Some are political interest-groups. But every single group +has that same quality of apparent spontaneity--"Hey, gang! +My uncle's got a barn--let's put on a show!" + +Every one of these groups is embarrassed by this "amateurism," +and, for the sake of their public image in a world of non-computer people, +they all attempt to look as stern and formal and impressive as possible. +These electronic frontier-dwellers resemble groups of nineteenth-century +pioneers hankering after the respectability of statehood. +There are however, two crucial differences in the historical experience +of these "pioneers" of the nineteeth and twenty-first centuries. + +First, powerful information technology DOES play into the hands of small, +fluid, loosely organized groups. There have always been "pioneers," +"hobbyists," "amateurs," "dilettantes," "volunteers," "movements," +"users' groups" and "blue-ribbon panels of experts" around. +But a group of this kind--when technically equipped to ship +huge amounts of specialized information, at lightning speed, +to its members, to government, and to the press--is simply +a different kind of animal. It's like the difference between +an eel and an electric eel. + +The second crucial change is that American society is currently +in a state approaching permanent technological revolution. +In the world of computers particularly, it is practically impossible +to EVER stop being a "pioneer," unless you either drop dead or +deliberately jump off the bus. The scene has never slowed down +enough to become well-institutionalized. And after twenty, thirty, +forty years the "computer revolution" continues to spread, +to permeate new corners of society. Anything that really works +is already obsolete. + +If you spend your entire working life as a "pioneer," the word "pioneer" +begins to lose its meaning. Your way of life looks less and less like +an introduction to something else" more stable and organized, +and more and more like JUST THE WAY THINGS ARE. A "permanent revolution" +is really a contradiction in terms. If "turmoil" lasts long enough, +it simply becomes A NEW KIND OF SOCIETY--still the same game of history, +but new players, new rules. + +Apply this to the world of late twentieth-century law enforcement, +and the implications are novel and puzzling indeed. Any bureaucratic +rulebook you write about computer-crime will be flawed when you write it, +and almost an antique by the time it sees print. The fluidity and fast +reactions of the FCIC give them a great advantage in this regard, +which explains their success. Even with the best will in the world +(which it does not, in fact, possess) it is impossible for an organization +the size of the U.S. Federal Bureau of Investigation to get up to speed +on the theory and practice of computer crime. If they tried to train all +their agents to do this, it would be SUICIDAL, as they would NEVER BE ABLE +TO DO ANYTHING ELSE. + +The FBI does try to train its agents in the basics of electronic crime, +at their base in Quantico, Virginia. And the Secret Service, along with +many other law enforcement groups, runs quite successful and well-attended +training courses on wire fraud, business crime, and computer intrusion +at the Federal Law Enforcement Training Center (FLETC, pronounced "fletsy") +in Glynco, Georgia. But the best efforts of these bureaucracies does not +remove the absolute need for a "cutting-edge mess" like the FCIC. + +For you see--the members of FCIC ARE the trainers of the rest +of law enforcement. Practically and literally speaking, +they are the Glynco computer-crime faculty by another name. +If the FCIC went over a cliff on a bus, the U.S. law enforcement +community would be rendered deaf dumb and blind in the world +of computer crime, and would swiftly feel a desperate need +to reinvent them. And this is no time to go starting from scratch. + +On June 11, 1991, I once again arrived in Phoenix, Arizona, +for the latest meeting of the Federal Computer Investigations Committee. +This was more or less the twentieth meeting of this stellar group. +The count was uncertain, since nobody could figure out whether to +include the meetings of "the Colluquy," which is what the FCIC +was called in the mid-1980s before it had even managed to obtain +the dignity of its own acronym. + +Since my last visit to Arizona, in May, the local AzScam bribery scandal +had resolved itself in a general muddle of humiliation. The Phoenix chief +of police, whose agents had videotaped nine state legislators up to no good, +had resigned his office in a tussle with the Phoenix city council over +the propriety of his undercover operations. + +The Phoenix Chief could now join Gail Thackeray and eleven of her closest +associates in the shared experience of politically motivated unemployment. +As of June, resignations were still continuing at the Arizona Attorney +General's office, which could be interpreted as either a New Broom +Sweeping Clean or a Night of the Long Knives Part II, depending on +your point of view. + +The meeting of FCIC was held at the Scottsdale Hilton Resort. +Scottsdale is a wealthy suburb of Phoenix, known as "Scottsdull" +to scoffing local trendies, but well-equipped with posh shopping-malls +and manicured lawns, while conspicuously undersupplied with homeless derelicts. +The Scottsdale Hilton Resort was a sprawling hotel in postmodern +crypto-Southwestern style. It featured a "mission bell tower" +plated in turquoise tile and vaguely resembling a Saudi minaret. + +Inside it was all barbarically striped Santa Fe Style decor. +There was a health spa downstairs and a large oddly-shaped +pool in the patio. A poolside umbrella-stand offered Ben and Jerry's +politically correct Peace Pops. + +I registered as a member of FCIC, attaining a handy discount rate, +then went in search of the Feds. Sure enough, at the back of the +hotel grounds came the unmistakable sound of Gail Thackeray +holding forth. + +Since I had also attended the Computers Freedom and Privacy conference +(about which more later), this was the second time I had seen Thackeray +in a group of her law enforcement colleagues. Once again I was struck +by how simply pleased they seemed to see her. It was natural that she'd +get SOME attention, as Gail was one of two women in a group of some thirty men; +but there was a lot more to it than that. + +Gail Thackeray personifies the social glue of the FCIC. They could give +a damn about her losing her job with the Attorney General. They were sorry +about it, of course, but hell, they'd all lost jobs. If they were the kind +of guys who liked steady boring jobs, they would never have gotten into +computer work in the first place. + +I wandered into her circle and was immediately introduced to five strangers. +The conditions of my visit at FCIC were reviewed. I would not quote +anyone directly. I would not tie opinions expressed to the agencies +of the attendees. I would not (a purely hypothetical example) +report the conversation of a guy from the Secret Service talking +quite civilly to a guy from the FBI, as these two agencies NEVER +talk to each other, and the IRS (also present, also hypothetical) +NEVER TALKS TO ANYBODY. + +Worse yet, I was forbidden to attend the first conference. And I didn't. +I have no idea what the FCIC was up to behind closed doors that afternoon. +I rather suspect that they were engaging in a frank and thorough confession +of their errors, goof-ups and blunders, as this has been a feature of every +FCIC meeting since their legendary Memphis beer-bust of 1986. Perhaps the +single greatest attraction of FCIC is that it is a place where you can go, +let your hair down, and completely level with people who actually comprehend +what you are talking about. Not only do they understand you, but they +REALLY PAY ATTENTION, they are GRATEFUL FOR YOUR INSIGHTS, and they +FORGIVE YOU, which in nine cases out of ten is something even your +boss can't do, because as soon as you start talking "ROM," "BBS," +or "T-1 trunk," his eyes glaze over. + +I had nothing much to do that afternoon. The FCIC were beavering away +in their conference room. Doors were firmly closed, windows too dark +to peer through. I wondered what a real hacker, a computer intruder, +would do at a meeting like this. + +The answer came at once. He would "trash" the place. Not reduce the place +to trash in some orgy of vandalism; that's not the use of the term in the +hacker milieu. No, he would quietly EMPTY THE TRASH BASKETS and silently +raid any valuable data indiscreetly thrown away. + +Journalists have been known to do this. (Journalists hunting information +have been known to do almost every single unethical thing that hackers +have ever done. They also throw in a few awful techniques all their own.) +The legality of `trashing' is somewhat dubious but it is not in fact +flagrantly illegal. It was, however, absurd to contemplate trashing the FCIC. +These people knew all about trashing. I wouldn't last fifteen seconds. + +The idea sounded interesting, though. I'd been hearing a lot about +the practice lately. On the spur of the moment, I decided I would try +trashing the office ACROSS THE HALL from the FCIC, an area which had +nothing to do with the investigators. + +The office was tiny; six chairs, a table. . . . Nevertheless, it was open, +so I dug around in its plastic trash can. + +To my utter astonishment, I came up with the torn scraps of a SPRINT +long-distance phone bill. More digging produced a bank statement +and the scraps of a hand-written letter, along with gum, cigarette ashes, +candy wrappers and a day-old-issue of USA TODAY. + +The trash went back in its receptacle while the scraps of data went into +my travel bag. I detoured through the hotel souvenir shop for some +Scotch tape and went up to my room. + +Coincidence or not, it was quite true. Some poor soul had, in fact, +thrown a SPRINT bill into the hotel's trash. Date May 1991, +total amount due: $252.36. Not a business phone, either, +but a residential bill, in the name of someone called Evelyn +(not her real name). Evelyn's records showed a ## PAST DUE BILL ##! +Here was her nine-digit account ID. Here was a stern computer-printed warning: + +"TREAT YOUR FONCARD AS YOU WOULD ANY CREDIT CARD. TO SECURE AGAINST FRAUD, +NEVER GIVE YOUR FONCARD NUMBER OVER THE PHONE UNLESS YOU INITIATED THE CALL. +IF YOU RECEIVE SUSPICIOUS CALLS PLEASE NOTIFY CUSTOMER SERVICE IMMEDIATELY!" + +I examined my watch. Still plenty of time left for the FCIC to carry on. +I sorted out the scraps of Evelyn's SPRINT bill and re-assembled them with +fresh Scotch tape. Here was her ten-digit FONCARD number. Didn't seem +to have the ID number necessary to cause real fraud trouble. + +I did, however, have Evelyn's home phone number. And the phone numbers +for a whole crowd of Evelyn's long-distance friends and acquaintances. +In San Diego, Folsom, Redondo, Las Vegas, La Jolla, Topeka, and Northampton +Massachusetts. Even somebody in Australia! + +I examined other documents. Here was a bank statement. It was Evelyn's +IRA account down at a bank in San Mateo California (total balance $1877.20). +Here was a charge-card bill for $382.64. She was paying it off bit by bit. + +Driven by motives that were completely unethical and prurient, +I now examined the handwritten notes. They had been torn fairly +thoroughly, so much so that it took me almost an entire five minutes +to reassemble them. + +They were drafts of a love letter. They had been written on +the lined stationery of Evelyn's employer, a biomedical company. +Probably written at work when she should have been doing something else. + +"Dear Bob," (not his real name) "I guess in everyone's life there comes +a time when hard decisions have to be made, and this is a difficult one +for me--very upsetting. Since you haven't called me, and I don't understand +why, I can only surmise it's because you don't want to. I thought I would +have heard from you Friday. I did have a few unusual problems with my phone +and possibly you tried, I hope so. + +"Robert, you asked me to `let go'. . . ." + +The first note ended. UNUSUAL PROBLEMS WITH HER PHONE? +I looked swiftly at the next note. + +"Bob, not hearing from you for the whole weekend has left me very perplexed. . . ." + +Next draft. + +"Dear Bob, there is so much I don't understand right now, and I wish I did. +I wish I could talk to you, but for some unknown reason you have elected not +to call--this is so difficult for me to understand. . . ." + +She tried again. + +"Bob, Since I have always held you in such high esteem, I had every hope that +we could remain good friends, but now one essential ingredient is missing-- +respect. Your ability to discard people when their purpose is served is +appalling to me. The kindest thing you could do for me now is to leave me +alone. You are no longer welcome in my heart or home. . . ." + +Try again. + +"Bob, I wrote a very factual note to you to say how much respect I had lost +for you, by the way you treat people, me in particular, so uncaring and cold. +The kindest thing you can do for me is to leave me alone entirely, +as you are no longer welcome in my heart or home. I would appreciate it +if you could retire your debt to me as soon as possible--I wish no link +to you in any way. Sincerely, Evelyn." + +Good heavens, I thought, the bastard actually owes her money! +I turned to the next page. + +"Bob: very simple. GOODBYE! No more mind games--no more fascination-- +no more coldness--no more respect for you! It's over--Finis. Evie" + +There were two versions of the final brushoff letter, but they read about +the same. Maybe she hadn't sent it. The final item in my illicit and +shameful booty was an envelope addressed to "Bob" at his home address, +but it had no stamp on it and it hadn't been mailed. + +Maybe she'd just been blowing off steam because her rascal boyfriend +had neglected to call her one weekend. Big deal. Maybe they'd kissed +and made up, maybe she and Bob were down at Pop's Chocolate Shop now, +sharing a malted. Sure. + +Easy to find out. All I had to do was call Evelyn up. With a half-clever +story and enough brass-plated gall I could probably trick the truth out of her. +Phone-phreaks and hackers deceive people over the phone all the time. +It's called "social engineering." Social engineering is a very common practice +in the underground, and almost magically effective. Human beings are almost +always the weakest link in computer security. The simplest way to learn +Things You Are Not Meant To Know is simply to call up and exploit the +knowledgeable people. With social engineering, you use the bits of specialized +knowledge you already have as a key, to manipulate people into believing +that you are legitimate. You can then coax, flatter, or frighten them into +revealing almost anything you want to know. Deceiving people (especially +over the phone) is easy and fun. Exploiting their gullibility is very +gratifying; it makes you feel very superior to them. + +If I'd been a malicious hacker on a trashing raid, I would now have Evelyn +very much in my power. Given all this inside data, it wouldn't take much +effort at all to invent a convincing lie. If I were ruthless enough, +and jaded enough, and clever enough, this momentary indiscretion of hers-- +maybe committed in tears, who knows--could cause her a whole world of +confusion and grief. + +I didn't even have to have a MALICIOUS motive. Maybe I'd be "on her side," +and call up Bob instead, and anonymously threaten to break both his kneecaps +if he didn't take Evelyn out for a steak dinner pronto. It was still +profoundly NONE OF MY BUSINESS. To have gotten this knowledge at all +was a sordid act and to use it would be to inflict a sordid injury. + +To do all these awful things would require exactly zero high-tech expertise. +All it would take was the willingness to do it and a certain amount +of bent imagination. + +I went back downstairs. The hard-working FCIC, who had labored forty-five +minutes over their schedule, were through for the day, and adjourned to the +hotel bar. We all had a beer. + +I had a chat with a guy about "Isis," or rather IACIS, +the International Association of Computer Investigation Specialists. +They're into "computer forensics," the techniques of picking computer- +systems apart without destroying vital evidence. IACIS, currently run +out of Oregon, is comprised of investigators in the U.S., Canada, Taiwan +and Ireland. "Taiwan and Ireland?" I said. Are TAIWAN and IRELAND +really in the forefront of this stuff? Well not exactly, my informant +admitted. They just happen to have been the first ones to have caught +on by word of mouth. Still, the international angle counts, because this +is obviously an international problem. Phone-lines go everywhere. + +There was a Mountie here from the Royal Canadian Mounted Police. +He seemed to be having quite a good time. Nobody had flung this +Canadian out because he might pose a foreign security risk. +These are cyberspace cops. They still worry a lot about "jurisdictions," +but mere geography is the least of their troubles. + +NASA had failed to show. NASA suffers a lot from computer intrusions, +in particular from Australian raiders and a well-trumpeted Chaos +Computer Club case, and in 1990 there was a brief press flurry +when it was revealed that one of NASA's Houston branch-exchanges +had been systematically ripped off by a gang of phone-phreaks. +But the NASA guys had had their funding cut. They were stripping everything. + +Air Force OSI, its Office of Special Investigations, is the ONLY federal +entity dedicated full-time to computer security. They'd been expected +to show up in force, but some of them had cancelled--a Pentagon budget pinch. + +As the empties piled up, the guys began joshing around and telling war-stories. +"These are cops," Thackeray said tolerantly. "If they're not talking shop +they talk about women and beer." + +I heard the story about the guy who, asked for "a copy" of a computer disk, +PHOTOCOPIED THE LABEL ON IT. He put the floppy disk onto the glass plate +of a photocopier. The blast of static when the copier worked completely +erased all the real information on the disk. + +Some other poor souls threw a whole bag of confiscated diskettes +into the squad-car trunk next to the police radio. The powerful radio +signal blasted them, too. + +We heard a bit about Dave Geneson, the first computer prosecutor, +a mainframe-runner in Dade County, turned lawyer. Dave Geneson +was one guy who had hit the ground running, a signal virtue +in making the transition to computer-crime. It was generally +agreed that it was easier to learn the world of computers first, +then police or prosecutorial work. You could take certain computer +people and train 'em to successful police work--but of course they +had to have the COP MENTALITY. They had to have street smarts. +Patience. Persistence. And discretion. You've got to make sure +they're not hot-shots, show-offs, "cowboys." + +Most of the folks in the bar had backgrounds in military intelligence, +or drugs, or homicide. It was rudely opined that "military intelligence" +was a contradiction in terms, while even the grisly world of homicide +was considered cleaner than drug enforcement. One guy had been 'way +undercover doing dope-work in Europe for four years straight. +"I'm almost recovered now," he said deadpan, with the acid black humor +that is pure cop. "Hey, now I can say FUCKER without putting MOTHER +in front of it." + +"In the cop world," another guy said earnestly, "everything is good and bad, +black and white. In the computer world everything is gray." + +One guy--a founder of the FCIC, who'd been with the group +since it was just the Colluquy--described his own introduction +to the field. He'd been a Washington DC homicide guy called in +on a "hacker" case. From the word "hacker," he naturally assumed +he was on the trail of a knife-wielding marauder, and went to the +computer center expecting blood and a body. When he finally figured +out what was happening there (after loudly demanding, in vain, +that the programmers "speak English"), he called headquarters +and told them he was clueless about computers. They told him nobody +else knew diddly either, and to get the hell back to work. + +So, he said, he had proceeded by comparisons. By analogy. By metaphor. +"Somebody broke in to your computer, huh?" Breaking and entering; +I can understand that. How'd he get in? "Over the phone-lines." +Harassing phone-calls, I can understand that! What we need here +is a tap and a trace! + +It worked. It was better than nothing. And it worked a lot faster +when he got hold of another cop who'd done something similar. +And then the two of them got another, and another, and pretty soon +the Colluquy was a happening thing. It helped a lot that everybody +seemed to know Carlton Fitzpatrick, the data-processing trainer in Glynco. + +The ice broke big-time in Memphis in '86. The Colluquy had attracted +a bunch of new guys--Secret Service, FBI, military, other feds, heavy guys. +Nobody wanted to tell anybody anything. They suspected that if word got back +to the home office they'd all be fired. They passed an uncomfortably +guarded afternoon. + +The formalities got them nowhere. But after the formal session was over, +the organizers brought in a case of beer. As soon as the participants +knocked it off with the bureaucratic ranks and turf-fighting, everything +changed. "I bared my soul," one veteran reminisced proudly. By nightfall +they were building pyramids of empty beer-cans and doing everything +but composing a team fight song. + +FCIC were not the only computer-crime people around. There was DATTA +(District Attorneys' Technology Theft Association), though they mostly +specialized in chip theft, intellectual property, and black-market cases. +There was HTCIA (High Tech Computer Investigators Association), +also out in Silicon Valley, a year older than FCIC and featuring +brilliant people like Donald Ingraham. There was LEETAC +(Law Enforcement Electronic Technology Assistance Committee) +in Florida, and computer-crime units in Illinois and Maryland +and Texas and Ohio and Colorado and Pennsylvania. But these were +local groups. FCIC were the first to really network nationally +and on a federal level. + +FCIC people live on the phone lines. Not on bulletin board systems-- +they know very well what boards are, and they know that boards aren't secure. +Everyone in the FCIC has a voice-phone bill like you wouldn't believe. +FCIC people have been tight with the telco people for a long time. +Telephone cyberspace is their native habitat. + +FCIC has three basic sub-tribes: the trainers, the security people, +and the investigators. That's why it's called an "Investigations +Committee" with no mention of the term "computer-crime"--the dreaded +"C-word." FCIC, officially, is "an association of agencies rather +than individuals;" unofficially, this field is small enough that +the influence of individuals and individual expertise is paramount. +Attendance is by invitation only, and most everyone in FCIC considers +himself a prophet without honor in his own house. + +Again and again I heard this, with different terms but identical +sentiments. "I'd been sitting in the wilderness talking to myself." +"I was totally isolated." "I was desperate." "FCIC is the best +thing there is about computer crime in America." "FCIC is what +really works." "This is where you hear real people telling you +what's really happening out there, not just lawyers picking nits." +"We taught each other everything we knew." + +The sincerity of these statements convinces me that this is true. +FCIC is the real thing and it is invaluable. It's also very sharply +at odds with the rest of the traditions and power structure +in American law enforcement. There probably hasn't been anything +around as loose and go-getting as the FCIC since the start of the +U.S. Secret Service in the 1860s. FCIC people are living like +twenty-first-century people in a twentieth-century environment, +and while there's a great deal to be said for that, there's also +a great deal to be said against it, and those against it happen +to control the budgets. + +I listened to two FCIC guys from Jersey compare life histories. +One of them had been a biker in a fairly heavy-duty gang in the 1960s. +"Oh, did you know so-and-so?" said the other guy from Jersey. +"Big guy, heavyset?" + +"Yeah, I knew him." + +"Yeah, he was one of ours. He was our plant in the gang." + +"Really? Wow! Yeah, I knew him. Helluva guy." + +Thackeray reminisced at length about being tear-gassed blind +in the November 1969 antiwar protests in Washington Circle, +covering them for her college paper. "Oh yeah, I was there," +said another cop. "Glad to hear that tear gas hit somethin'. +Haw haw haw." He'd been so blind himself, he confessed, +that later that day he'd arrested a small tree. + +FCIC are an odd group, sifted out by coincidence and necessity, +and turned into a new kind of cop. There are a lot of specialized +cops in the world--your bunco guys, your drug guys, your tax guys, +but the only group that matches FCIC for sheer isolation are probably +the child-pornography people. Because they both deal with conspirators +who are desperate to exchange forbidden data and also desperate to hide; +and because nobody else in law enforcement even wants to hear about it. + +FCIC people tend to change jobs a lot. They tend not to get the equipment +and training they want and need. And they tend to get sued quite often. + +As the night wore on and a band set up in the bar, the talk grew darker. +Nothing ever gets done in government, someone opined, until there's +a DISASTER. Computing disasters are awful, but there's no denying +that they greatly help the credibility of FCIC people. The Internet Worm, +for instance. "For years we'd been warning about that--but it's nothing +compared to what's coming." They expect horrors, these people. +They know that nothing will really get done until there is a horror. + +# + +Next day we heard an extensive briefing from a guy who'd been a computer cop, +gotten into hot water with an Arizona city council, and now installed +computer networks for a living (at a considerable rise in pay). +He talked about pulling fiber-optic networks apart. + +Even a single computer, with enough peripherals, is a literal +"network"--a bunch of machines all cabled together, generally +with a complexity that puts stereo units to shame. FCIC people +invent and publicize methods of seizing computers and maintaining +their evidence. Simple things, sometimes, but vital rules of thumb +for street cops, who nowadays often stumble across a busy computer +in the midst of a drug investigation or a white-collar bust. +For instance: Photograph the system before you touch it. +Label the ends of all the cables before you detach anything. +"Park" the heads on the disk drives before you move them. +Get the diskettes. Don't put the diskettes in magnetic fields. +Don't write on diskettes with ballpoint pens. Get the manuals. +Get the printouts. Get the handwritten notes. Copy data before +you look at it, and then examine the copy instead of the original. + +Now our lecturer distributed copied diagrams of a typical LAN +or "Local Area Network", which happened to be out of Connecticut. +ONE HUNDRED AND FIFTY-NINE desktop computers, each with its own +peripherals. Three "file servers." Five "star couplers" +each with thirty-two ports. One sixteen-port coupler +off in the corner office. All these machines talking to each other, +distributing electronic mail, distributing software, distributing, +quite possibly, criminal evidence. All linked by high-capacity +fiber-optic cable. A bad guy--cops talk a about "bad guys" +--might be lurking on PC #47 lot or #123 and distributing +his ill doings onto some dupe's "personal" machine in +another office--or another floor--or, quite possibly, +two or three miles away! Or, conceivably, the evidence might +be "data-striped"--split up into meaningless slivers stored, +one by one, on a whole crowd of different disk drives. + +The lecturer challenged us for solutions. I for one was utterly clueless. +As far as I could figure, the Cossacks were at the gate; there were probably +more disks in this single building than were seized during the entirety +of Operation Sundevil. + +"Inside informant," somebody said. Right. There's always the human angle, +something easy to forget when contemplating the arcane recesses of high +technology. Cops are skilled at getting people to talk, and computer people, +given a chair and some sustained attention, will talk about their computers +till their throats go raw. There's a case on record of a single question-- +"How'd you do it?"--eliciting a forty-five-minute videotaped confession +from a computer criminal who not only completely incriminated himself +but drew helpful diagrams. + +Computer people talk. Hackers BRAG. Phone-phreaks +talk PATHOLOGICALLY--why else are they stealing phone-codes, +if not to natter for ten hours straight to their friends +on an opposite seaboard? Computer-literate people do +in fact possess an arsenal of nifty gadgets and techniques +that would allow them to conceal all kinds of exotic skullduggery, +and if they could only SHUT UP about it, they could probably +get away with all manner of amazing information-crimes. +But that's just not how it works--or at least, +that's not how it's worked SO FAR. + +Most every phone-phreak ever busted has swiftly implicated his mentors, +his disciples, and his friends. Most every white-collar computer-criminal, +smugly convinced that his clever scheme is bulletproof, swiftly learns +otherwise when, for the first time in his life, an actual no-kidding +policeman leans over, grabs the front of his shirt, looks him right +in the eye and says: "All right, ASSHOLE--you and me are going downtown!" +All the hardware in the world will not insulate your nerves from +these actual real-life sensations of terror and guilt. + +Cops know ways to get from point A to point Z without thumbing +through every letter in some smart-ass bad-guy's alphabet. +Cops know how to cut to the chase. Cops know a lot of things +other people don't know. + +Hackers know a lot of things other people don't know, too. +Hackers know, for instance, how to sneak into your computer +through the phone-lines. But cops can show up RIGHT ON YOUR DOORSTEP +and carry off YOU and your computer in separate steel boxes. +A cop interested in hackers can grab them and grill them. +A hacker interested in cops has to depend on hearsay, +underground legends, and what cops are willing to publicly reveal. +And the Secret Service didn't get named "the SECRET Service" +because they blab a lot. + +Some people, our lecturer informed us, were under the mistaken +impression that it was "impossible" to tap a fiber-optic line. +Well, he announced, he and his son had just whipped up a +fiber-optic tap in his workshop at home. He passed it around +the audience, along with a circuit-covered LAN plug-in card +so we'd all recognize one if we saw it on a case. We all had a look. + +The tap was a classic "Goofy Prototype"--a thumb-length rounded +metal cylinder with a pair of plastic brackets on it. +From one end dangled three thin black cables, each of which ended +in a tiny black plastic cap. When you plucked the safety-cap +off the end of a cable, you could see the glass fiber-- +no thicker than a pinhole. + +Our lecturer informed us that the metal cylinder was a +"wavelength division multiplexer." Apparently, what one did +was to cut the fiber-optic cable, insert two of the legs into +the cut to complete the network again, and then read any passing data +on the line by hooking up the third leg to some kind of monitor. +Sounded simple enough. I wondered why nobody had thought of it before. +I also wondered whether this guy's son back at the workshop had any +teenage friends. + +We had a break. The guy sitting next to me was wearing a giveaway +baseball cap advertising the Uzi submachine gun. We had a desultory chat +about the merits of Uzis. Long a favorite of the Secret Service, +it seems Uzis went out of fashion with the advent of the Persian Gulf War, +our Arab allies taking some offense at Americans toting Israeli weapons. +Besides, I was informed by another expert, Uzis jam. The equivalent weapon +of choice today is the Heckler & Koch, manufactured in Germany. + +The guy with the Uzi cap was a forensic photographer. He also did a lot +of photographic surveillance work in computer crime cases. He used to, +that is, until the firings in Phoenix. He was now a private investigator and, +with his wife, ran a photography salon specializing in weddings and portrait +photos. At--one must repeat--a considerable rise in income. + +He was still FCIC. If you were FCIC, and you needed to talk +to an expert about forensic photography, well, there he was, +willing and able. If he hadn't shown up, people would have missed him. + +Our lecturer had raised the point that preliminary investigation +of a computer system is vital before any seizure is undertaken. +It's vital to understand how many machines are in there, what kinds +there are, what kind of operating system they use, how many people +use them, where the actual data itself is stored. To simply barge into +an office demanding "all the computers" is a recipe for swift disaster. + +This entails some discreet inquiries beforehand. In fact, what it +entails is basically undercover work. An intelligence operation. +SPYING, not to put too fine a point on it. + +In a chat after the lecture, I asked an attendee whether "trashing" might work. + +I received a swift briefing on the theory and practice of "trash covers." +Police "trash covers," like "mail covers" or like wiretaps, require the +agreement of a judge. This obtained, the "trashing" work of cops is just +like that of hackers, only more so and much better organized. So much so, +I was informed, that mobsters in Phoenix make extensive use of locked +garbage cans picked up by a specialty high-security trash company. + +In one case, a tiger team of Arizona cops had trashed a local residence +for four months. Every week they showed up on the municipal garbage truck, +disguised as garbagemen, and carried the contents of the suspect cans off +to a shade tree, where they combed through the garbage--a messy task, +especially considering that one of the occupants was undergoing +kidney dialysis. All useful documents were cleaned, dried and examined. +A discarded typewriter-ribbon was an especially valuable source of data, +as its long one-strike ribbon of film contained the contents of every +letter mailed out of the house. The letters were neatly retyped by +a police secretary equipped with a large desk-mounted magnifying glass. + +There is something weirdly disquieting about the whole subject of +"trashing"-- an unsuspected and indeed rather disgusting mode of +deep personal vulnerability. Things that we pass by every day, +that we take utterly for granted, can be exploited with so little work. +Once discovered, the knowledge of these vulnerabilities tend to spread. + +Take the lowly subject of MANHOLE COVERS. The humble manhole cover +reproduces many of the dilemmas of computer-security in miniature. +Manhole covers are, of course, technological artifacts, access-points +to our buried urban infrastructure. To the vast majority of us, +manhole covers are invisible. They are also vulnerable. For many years now, +the Secret Service has made a point of caulking manhole covers along all routes +of the Presidential motorcade. This is, of course, to deter terrorists from +leaping out of underground ambush or, more likely, planting remote-control +car-smashing bombs beneath the street. + +Lately, manhole covers have seen more and more criminal exploitation, +especially in New York City. Recently, a telco in New York City +discovered that a cable television service had been sneaking into +telco manholes and installing cable service alongside the phone-lines-- +WITHOUT PAYING ROYALTIES. New York companies have also suffered a +general plague of (a) underground copper cable theft; (b) dumping of garbage, +including toxic waste, and (c) hasty dumping of murder victims. + +Industry complaints reached the ears of an innovative New England +industrial-security company, and the result was a new product known +as "the Intimidator," a thick titanium-steel bolt with a precisely machined +head that requires a special device to unscrew. All these "keys" have registered +serial numbers kept on file with the manufacturer. There are now some +thousands of these "Intimidator" bolts being sunk into American pavements +wherever our President passes, like some macabre parody of strewn roses. +They are also spreading as fast as steel dandelions around US military bases +and many centers of private industry. + +Quite likely it has never occurred to you to peer under a manhole cover, +perhaps climb down and walk around down there with a flashlight, just to see +what it's like. Formally speaking, this might be trespassing, but if you +didn't hurt anything, and didn't make an absolute habit of it, nobody would +really care. The freedom to sneak under manholes was likely a freedom +you never intended to exercise. + +You now are rather less likely to have that freedom at all. +You may never even have missed it until you read about it here, +but if you're in New York City it's gone, and elsewhere it's likely going. +This is one of the things that crime, and the reaction to +crime, does to us. + +The tenor of the meeting now changed as the Electronic Frontier Foundation +arrived. The EFF, whose personnel and history will be examined in detail +in the next chapter, are a pioneering civil liberties group who arose in +direct response to the Hacker Crackdown of 1990. + +Now Mitchell Kapor, the Foundation's president, and Michael Godwin, +its chief attorney, were confronting federal law enforcement MANO A MANO +for the first time ever. Ever alert to the manifold uses of publicity, +Mitch Kapor and Mike Godwin had brought their own journalist in tow: +Robert Draper, from Austin, whose recent well-received book about +ROLLING STONE magazine was still on the stands. Draper was on assignment +for TEXAS MONTHLY. + +The Steve Jackson/EFF civil lawsuit against the Chicago Computer Fraud +and Abuse Task Force was a matter of considerable regional interest in Texas. +There were now two Austinite journalists here on the case. In fact, +counting Godwin (a former Austinite and former journalist) there were +three of us. Lunch was like Old Home Week. + +Later, I took Draper up to my hotel room. We had a long frank talk +about the case, networking earnestly like a miniature freelance-journo +version of the FCIC: privately confessing the numerous blunders +of journalists covering the story, and trying hard to figure out +who was who and what the hell was really going on out there. +I showed Draper everything I had dug out of the Hilton trashcan. +We pondered the ethics of "trashing" for a while, and agreed +that they were dismal. We also agreed that finding a SPRINT +bill on your first time out was a heck of a coincidence. + +First I'd "trashed"--and now, mere hours later, I'd bragged to someone else. +Having entered the lifestyle of hackerdom, I was now, unsurprisingly, +following its logic. Having discovered something remarkable through +a surreptitious action, I of course HAD to "brag," and to drag the passing +Draper into my iniquities. I felt I needed a witness. Otherwise nobody +would have believed what I'd discovered. . . . + +Back at the meeting, Thackeray cordially, if rather tentatively, +introduced Kapor and Godwin to her colleagues. Papers were distributed. +Kapor took center stage. The brilliant Bostonian high-tech entrepreneur, +normally the hawk in his own administration and quite an effective +public speaker, seemed visibly nervous, and frankly admitted as much. +He began by saying he consided computer-intrusion to be morally wrong, +and that the EFF was not a "hacker defense fund," despite what had appeared +in print. Kapor chatted a bit about the basic motivations of his group, +emphasizing their good faith and willingness to listen and seek common ground +with law enforcement--when, er, possible. + +Then, at Godwin's urging, Kapor suddenly remarked that EFF's own Internet +machine had been "hacked" recently, and that EFF did not consider +this incident amusing. + +After this surprising confession, things began to loosen up +quite rapidly. Soon Kapor was fielding questions, parrying objections, +challenging definitions, and juggling paradigms with something akin +to his usual gusto. + +Kapor seemed to score quite an effect with his shrewd and skeptical analysis +of the merits of telco "Caller-ID" services. (On this topic, FCIC and EFF +have never been at loggerheads, and have no particular established earthworks +to defend.) Caller-ID has generally been promoted as a privacy service +for consumers, a presentation Kapor described as a "smokescreen," +the real point of Caller-ID being to ALLOW CORPORATE CUSTOMERS TO BUILD +EXTENSIVE COMMERCIAL DATABASES ON EVERYBODY WHO PHONES OR FAXES THEM. +Clearly, few people in the room had considered this possibility, +except perhaps for two late-arrivals from US WEST RBOC security, +who chuckled nervously. + +Mike Godwin then made an extensive presentation on +"Civil Liberties Implications of Computer Searches and Seizures." +Now, at last, we were getting to the real nitty-gritty here, +real political horse-trading. The audience listened with close +attention, angry mutters rising occasionally: "He's trying to +teach us our jobs!" "We've been thinking about this for years! +We think about these issues every day!" "If I didn't seize the works, +I'd be sued by the guy's victims!" "I'm violating the law if I leave +ten thousand disks full of illegal PIRATED SOFTWARE and STOLEN CODES!" +"It's our job to make sure people don't trash the Constitution-- +we're the DEFENDERS of the Constitution!" "We seize stuff when +we know it will be forfeited anyway as restitution for the victim!" + +"If it's forfeitable, then don't get a search warrant, get a +forfeiture warrant," Godwin suggested coolly. He further remarked +that most suspects in computer crime don't WANT to see their computers +vanish out the door, headed God knew where, for who knows how long. +They might not mind a search, even an extensive search, but they want +their machines searched on-site. + +"Are they gonna feed us?" somebody asked sourly. + +"How about if you take copies of the data?" Godwin parried. + +"That'll never stand up in court." + +"Okay, you make copies, give THEM the copies, and take the originals." + +Hmmm. + +Godwin championed bulletin-board systems as repositories of First Amendment +protected free speech. He complained that federal computer-crime training +manuals gave boards a bad press, suggesting that they are hotbeds of crime +haunted by pedophiles and crooks, whereas the vast majority of the nation's +thousands of boards are completely innocuous, and nowhere near so +romantically suspicious. + +People who run boards violently resent it when their systems are seized, +and their dozens (or hundreds) of users look on in abject horror. +Their rights of free expression are cut short. Their right to associate +with other people is infringed. And their privacy is violated as their +private electronic mail becomes police property. + +Not a soul spoke up to defend the practice of seizing boards. +The issue passed in chastened silence. Legal principles aside-- +(and those principles cannot be settled without laws passed or +court precedents)--seizing bulletin boards has become public-relations +poison for American computer police. + +And anyway, it's not entirely necessary. If you're a cop, you can get 'most +everything you need from a pirate board, just by using an inside informant. +Plenty of vigilantes--well, CONCERNED CITIZENS--will inform police the moment +they see a pirate board hit their area (and will tell the police all about it, +in such technical detail, actually, that you kinda wish they'd shut up). +They will happily supply police with extensive downloads or printouts. +It's IMPOSSIBLE to keep this fluid electronic information out of the +hands of police. + +Some people in the electronic community become enraged at the prospect +of cops "monitoring" bulletin boards. This does have touchy aspects, +as Secret Service people in particular examine bulletin boards with +some regularity. But to expect electronic police to be deaf dumb +and blind in regard to this particular medium rather flies in the face +of common sense. Police watch television, listen to radio, read newspapers +and magazines; why should the new medium of boards be different? +Cops can exercise the same access to electronic information +as everybody else. As we have seen, quite a few computer +police maintain THEIR OWN bulletin boards, including anti-hacker +"sting" boards, which have generally proven quite effective. + +As a final clincher, their Mountie friends in Canada (and colleagues +in Ireland and Taiwan) don't have First Amendment or American +constitutional restrictions, but they do have phone lines, +and can call any bulletin board in America whenever they please. +The same technological determinants that play into the hands of hackers, +phone phreaks and software pirates can play into the hands of police. +"Technological determinants" don't have ANY human allegiances. +They're not black or white, or Establishment or Underground, +or pro-or-anti anything. + +Godwin complained at length about what he called "the Clever Hobbyist +hypothesis" --the assumption that the "hacker" you're busting is clearly +a technical genius, and must therefore by searched with extreme thoroughness. +So: from the law's point of view, why risk missing anything? Take the works. +Take the guy's computer. Take his books. Take his notebooks. +Take the electronic drafts of his love letters. Take his Walkman. +Take his wife's computer. Take his dad's computer. Take his kid +sister's computer. Take his employer's computer. Take his compact disks-- +they MIGHT be CD-ROM disks, cunningly disguised as pop music. +Take his laser printer--he might have hidden something vital in the +printer's 5meg of memory. Take his software manuals and hardware +documentation. Take his science-fiction novels and his simulation- +gaming books. Take his Nintendo Game-Boy and his Pac-Man arcade game. +Take his answering machine, take his telephone out of the wall. +Take anything remotely suspicious. + +Godwin pointed out that most "hackers" are not, in fact, clever +genius hobbyists. Quite a few are crooks and grifters who don't +have much in the way of technical sophistication; just some rule-of-thumb +rip-off techniques. The same goes for most fifteen-year-olds who've +downloaded a code-scanning program from a pirate board. There's no +real need to seize everything in sight. It doesn't require an entire +computer system and ten thousand disks to prove a case in court. + +What if the computer is the instrumentality of a crime? someone demanded. + +Godwin admitted quietly that the doctrine of seizing the instrumentality +of a crime was pretty well established in the American legal system. + +The meeting broke up. Godwin and Kapor had to leave. Kapor was testifying +next morning before the Massachusetts Department Of Public Utility, +about ISDN narrowband wide-area networking. + +As soon as they were gone, Thackeray seemed elated. +She had taken a great risk with this. Her colleagues had not, +in fact, torn Kapor and Godwin's heads off. She was very proud of them, +and told them so. + +"Did you hear what Godwin said about INSTRUMENTALITY OF A CRIME?" +she exulted, to nobody in particular. "Wow, that means +MITCH ISN'T GOING TO SUE ME." + +# + +America's computer police are an interesting group. +As a social phenomenon they are far more interesting, +and far more important, than teenage phone phreaks +and computer hackers. First, they're older and wiser; +not dizzy hobbyists with leaky morals, but seasoned adult +professionals with all the responsibilities of public service. +And, unlike hackers, they possess not merely TECHNICAL +power alone, but heavy-duty legal and social authority. + +And, very interestingly, they are just as much at +sea in cyberspace as everyone else. They are not +happy about this. Police are authoritarian by nature, +and prefer to obey rules and precedents. (Even those police +who secretly enjoy a fast ride in rough territory will soberly +disclaim any "cowboy" attitude.) But in cyberspace there ARE +no rules and precedents. They are groundbreaking pioneers, +Cyberspace Rangers, whether they like it or not. + +In my opinion, any teenager enthralled by computers, +fascinated by the ins and outs of computer security, +and attracted by the lure of specialized forms of knowledge and power, +would do well to forget all about "hacking" and set his (or her) +sights on becoming a fed. Feds can trump hackers at almost every +single thing hackers do, including gathering intelligence, +undercover disguise, trashing, phone-tapping, building dossiers, +networking, and infiltrating computer systems--CRIMINAL computer systems. +Secret Service agents know more about phreaking, coding and carding +than most phreaks can find out in years, and when it comes to viruses, +break-ins, software bombs and trojan horses, Feds have direct access to red-hot +confidential information that is only vague rumor in the underground. + +And if it's an impressive public rep you're after, there are few people +in the world who can be so chillingly impressive as a well-trained, +well-armed United States Secret Service agent. + +Of course, a few personal sacrifices are necessary in order to obtain +that power and knowledge. First, you'll have the galling discipline +of belonging to a large organization; but the world of computer crime +is still so small, and so amazingly fast-moving, that it will remain +spectacularly fluid for years to come. The second sacrifice is that +you'll have to give up ripping people off. This is not a great loss. +Abstaining from the use of illegal drugs, also necessary, will be a boon +to your health. + +A career in computer security is not a bad choice for a young man +or woman today. The field will almost certainly expand drastically +in years to come. If you are a teenager today, by the time you +become a professional, the pioneers you have read about in this book +will be the grand old men and women of the field, swamped by their many +disciples and successors. Of course, some of them, like William P. Wood +of the 1865 Secret Service, may well be mangled in the whirring machinery +of legal controversy; but by the time you enter the computer-crime field, +it may have stabilized somewhat, while remaining entertainingly challenging. + +But you can't just have a badge. You have to win it. First, there's the +federal law enforcement training. And it's hard--it's a challenge. +A real challenge--not for wimps and rodents. + +Every Secret Service agent must complete gruelling courses at the +Federal Law Enforcement Training Center. (In fact, Secret Service +agents are periodically re-trained during their entire careers.) + +In order to get a glimpse of what this might be like, +I myself travelled to FLETC. + +# + +The Federal Law Enforcement Training Center is a 1500-acre facility +on Georgia's Atlantic coast. It's a milieu of marshgrass, seabirds, +damp, clinging sea-breezes, palmettos, mosquitos, and bats. +Until 1974, it was a Navy Air Base, and still features a working runway, +and some WWII vintage blockhouses and officers' quarters. +The Center has since benefitted by a forty-million-dollar retrofit, +but there's still enough forest and swamp on the facility for the +Border Patrol to put in tracking practice. + +As a town, "Glynco" scarcely exists. The nearest real town is Brunswick, +a few miles down Highway 17, where I stayed at the aptly named Marshview +Holiday Inn. I had Sunday dinner at a seafood restaurant called "Jinright's," +where I feasted on deep-fried alligator tail. This local favorite was +a heaped basket of bite-sized chunks of white, tender, almost fluffy +reptile meat, steaming in a peppered batter crust. Alligator makes +a culinary experience that's hard to forget, especially when liberally +basted with homemade cocktail sauce from a Jinright squeeze-bottle. + +The crowded clientele were tourists, fishermen, local black folks +in their Sunday best, and white Georgian locals who all seemed +to bear an uncanny resemblance to Georgia humorist Lewis Grizzard. + +The 2,400 students from 75 federal agencies who make up the FLETC +population scarcely seem to make a dent in the low-key local scene. +The students look like tourists, and the teachers seem to have taken +on much of the relaxed air of the Deep South. My host was Mr. Carlton +Fitzpatrick, the Program Coordinator of the Financial Fraud Institute. +Carlton Fitzpatrick is a mustached, sinewy, well-tanned Alabama native +somewhere near his late forties, with a fondness for chewing tobacco, +powerful computers, and salty, down-home homilies. We'd met before, +at FCIC in Arizona. + +The Financial Fraud Institute is one of the nine divisions at FLETC. +Besides Financial Fraud, there's Driver & Marine, Firearms, +and Physical Training. These are specialized pursuits. +There are also five general training divisions: Basic Training, +Operations, Enforcement Techniques, Legal Division, and Behavioral Science. + +Somewhere in this curriculum is everything necessary to turn green college +graduates into federal agents. First they're given ID cards. Then they get +the rather miserable-looking blue coveralls known as "smurf suits." +The trainees are assigned a barracks and a cafeteria, and immediately +set on FLETC's bone-grinding physical training routine. Besides the +obligatory daily jogging--(the trainers run up danger flags beside +the track when the humidity rises high enough to threaten heat stroke)-- +here's the Nautilus machines, the martial arts, the survival skills. . . . + +The eighteen federal agencies who maintain on-site academies at FLETC +employ a wide variety of specialized law enforcement units, some of them +rather arcane. There's Border Patrol, IRS Criminal Investigation Division, +Park Service, Fish and Wildlife, Customs, Immigration, Secret Service and +the Treasury's uniformed subdivisions. . . . If you're a federal cop +and you don't work for the FBI, you train at FLETC. This includes people +as apparently obscure as the agents of the Railroad Retirement Board +Inspector General. Or the Tennessee Valley Authority Police, +who are in fact federal police officers, and can and do arrest criminals +on the federal property of the Tennessee Valley Authority. + +And then there are the computer-crime people. All sorts, all backgrounds. +Mr. Fitzpatrick is not jealous of his specialized knowledge. Cops all over, +in every branch of service, may feel a need to learn what he can teach. +Backgrounds don't matter much. Fitzpatrick himself was originally a +Border Patrol veteran, then became a Border Patrol instructor at FLETC. +His Spanish is still fluent--but he found himself strangely fascinated +when the first computers showed up at the Training Center. Fitzpatrick +did have a background in electrical engineering, and though he never +considered himself a computer hacker, he somehow found himself writing +useful little programs for this new and promising gizmo. + +He began looking into the general subject of computers and crime, +reading Donn Parker's books and articles, keeping an ear cocked +for war stories, useful insights from the field, the up-and-coming +people of the local computer-crime and high-technology units. . . . +Soon he got a reputation around FLETC as the resident "computer expert," +and that reputation alone brought him more exposure, more experience-- +until one day he looked around, and sure enough he WAS a federal +computer-crime expert. + +In fact, this unassuming, genial man may be THE federal computer-crime expert. +There are plenty of very good computer people, and plenty of very good +federal investigators, but the area where these worlds of expertise overlap +is very slim. And Carlton Fitzpatrick has been right at the center of that +since 1985, the first year of the Colluquy, a group which owes much to +his influence. + +He seems quite at home in his modest, acoustic-tiled office, +with its Ansel Adams-style Western photographic art, a gold-framed +Senior Instructor Certificate, and a towering bookcase crammed with +three-ring binders with ominous titles such as Datapro Reports on +Information Security and CFCA Telecom Security '90. + +The phone rings every ten minutes; colleagues show up at the door +to chat about new developments in locksmithing or to shake their heads +over the latest dismal developments in the BCCI global banking scandal. + +Carlton Fitzpatrick is a fount of computer-crime war-stories, +related in an acerbic drawl. He tells me the colorful tale +of a hacker caught in California some years back. He'd been +raiding systems, typing code without a detectable break, +for twenty, twenty-four, thirty-six hours straight. Not just +logged on--TYPING. Investigators were baffled. Nobody +could do that. Didn't he have to go to the bathroom? +Was it some kind of automatic keyboard-whacking device +that could actually type code? + +A raid on the suspect's home revealed a situation of astonishing squalor. +The hacker turned out to be a Pakistani computer-science student who had +flunked out of a California university. He'd gone completely underground +as an illegal electronic immigrant, and was selling stolen phone-service +to stay alive. The place was not merely messy and dirty, but in a state +of psychotic disorder. Powered by some weird mix of culture shock, +computer addiction, and amphetamines, the suspect had in fact been sitting +in front of his computer for a day and a half straight, with snacks and +drugs at hand on the edge of his desk and a chamber-pot under his chair. + +Word about stuff like this gets around in the hacker-tracker community. + +Carlton Fitzpatrick takes me for a guided tour by car around the +FLETC grounds. One of our first sights is the biggest indoor +firing range in the world. There are federal trainees in there, +Fitzpatrick assures me politely, blasting away with a wide variety +of automatic weapons: Uzis, Glocks, AK-47s. . . . He's willing to +take me inside. I tell him I'm sure that's really interesting, +but I'd rather see his computers. Carlton Fitzpatrick seems quite +surprised and pleased. I'm apparently the first journalist he's ever +seen who has turned down the shooting gallery in favor of microchips. + +Our next stop is a favorite with touring Congressmen: the three-mile +long FLETC driving range. Here trainees of the Driver & Marine Division +are taught high-speed pursuit skills, setting and breaking road-blocks, +diplomatic security driving for VIP limousines. . . . A favorite FLETC +pastime is to strap a passing Senator into the passenger seat beside a +Driver & Marine trainer, hit a hundred miles an hour, then take it right into +"the skid-pan," a section of greased track where two tons of Detroit iron +can whip and spin like a hockey puck. + +Cars don't fare well at FLETC. First they're rifled again and again +for search practice. Then they do 25,000 miles of high-speed +pursuit training; they get about seventy miles per set +of steel-belted radials. Then it's off to the skid pan, +where sometimes they roll and tumble headlong in the grease. +When they're sufficiently grease-stained, dented, and creaky, +they're sent to the roadblock unit, where they're battered without pity. +And finally then they're sacrificed to the Bureau of Alcohol, +Tobacco and Firearms, whose trainees learn the ins and outs +of car-bomb work by blowing them into smoking wreckage. + +There's a railroad box-car on the FLETC grounds, and a large +grounded boat, and a propless plane; all training-grounds for searches. +The plane sits forlornly on a patch of weedy tarmac next to an eerie +blockhouse known as the "ninja compound," where anti-terrorism specialists +practice hostage rescues. As I gaze on this creepy paragon of modern +low-intensity warfare, my nerves are jangled by a sudden staccato outburst +of automatic weapons fire, somewhere in the woods to my right. +"Nine-millimeter," Fitzpatrick judges calmly. + +Even the eldritch ninja compound pales somewhat compared +to the truly surreal area known as "the raid-houses." +This is a street lined on both sides with nondescript +concrete-block houses with flat pebbled roofs. +They were once officers' quarters. Now they are training grounds. +The first one to our left, Fitzpatrick tells me, has been specially +adapted for computer search-and-seizure practice. Inside it has been +wired for video from top to bottom, with eighteen pan-and-tilt +remotely controlled videocams mounted on walls and in corners. +Every movement of the trainee agent is recorded live by teachers, +for later taped analysis. Wasted movements, hesitations, possibly lethal +tactical mistakes--all are gone over in detail. + +Perhaps the weirdest single aspect of this building is its front door, +scarred and scuffed all along the bottom, from the repeated impact, +day after day, of federal shoe-leather. + +Down at the far end of the row of raid-houses some people are practicing +a murder. We drive by slowly as some very young and rather nervous-looking +federal trainees interview a heavyset bald man on the raid-house lawn. +Dealing with murder takes a lot of practice; first you have to learn +to control your own instinctive disgust and panic, then you have to learn +to control the reactions of a nerve-shredded crowd of civilians, +some of whom may have just lost a loved one, some of whom may be murderers-- +quite possibly both at once. + +A dummy plays the corpse. The roles of the bereaved, the morbidly curious, +and the homicidal are played, for pay, by local Georgians: waitresses, +musicians, most anybody who needs to moonlight and can learn a script. +These people, some of whom are FLETC regulars year after year, +must surely have one of the strangest jobs in the world. + +Something about the scene: "normal" people in a weird situation, +standing around talking in bright Georgia sunshine, unsuccessfully +pretending that something dreadful has gone on, while a dummy lies +inside on faked bloodstains. . . . While behind this weird masquerade, +like a nested set of Russian dolls, are grim future realities of real death, +real violence, real murders of real people, that these young agents +will really investigate, many times during their careers. . . . +Over and over. . . . Will those anticipated murders look like this, +feel like this--not as "real" as these amateur actors are trying to +make it seem, but both as "real," and as numbingly unreal, as watching +fake people standing around on a fake lawn? Something about this scene +unhinges me. It seems nightmarish to me, Kafkaesque. I simply don't +know how to take it; my head is turned around; I don't know whether to laugh, +cry, or just shudder. + +When the tour is over, Carlton Fitzpatrick and I talk about computers. +For the first time cyberspace seems like quite a comfortable place. +It seems very real to me suddenly, a place where I know what I'm talking about, +a place I'm used to. It's real. "Real." Whatever. + +Carlton Fitzpatrick is the only person I've met in cyberspace circles +who is happy with his present equipment. He's got a 5 Meg RAM PC with +a 112 meg hard disk; a 660 meg's on the way. He's got a Compaq 386 desktop, +and a Zenith 386 laptop with 120 meg. Down the hall is a NEC Multi-Sync 2A +with a CD-ROM drive and a 9600 baud modem with four com-lines. +There's a training minicomputer, and a 10-meg local mini just for the Center, +and a lab-full of student PC clones and half-a-dozen Macs or so. +There's a Data General MV 2500 with 8 meg on board and a 370 meg disk. + +Fitzpatrick plans to run a UNIX board on the Data General when he's +finished beta-testing the software for it, which he wrote himself. +It'll have E-mail features, massive files on all manner of computer-crime +and investigation procedures, and will follow the computer-security +specifics of the Department of Defense "Orange Book." He thinks +it will be the biggest BBS in the federal government. + +Will it have Phrack on it? I ask wryly. + +Sure, he tells me. Phrack, TAP, Computer Underground Digest, +all that stuff. With proper disclaimers, of course. + +I ask him if he plans to be the sysop. Running a system that size is very +time-consuming, and Fitzpatrick teaches two three-hour courses every day. + +No, he says seriously, FLETC has to get its money worth out of the instructors. +He thinks he can get a local volunteer to do it, a high-school student. + +He says a bit more, something I think about an Eagle Scout law-enforcement +liaison program, but my mind has rocketed off in disbelief. + +"You're going to put a TEENAGER in charge of a federal security BBS?" +I'm speechless. It hasn't escaped my notice that the FLETC Financial +Fraud Institute is the ULTIMATE hacker-trashing target; there is stuff in here, +stuff of such utter and consummate cool by every standard of the +digital underground. . . . + +I imagine the hackers of my acquaintance, fainting dead-away from +forbidden-knowledge greed-fits, at the mere prospect of cracking +the superultra top-secret computers used to train the Secret Service +in computer-crime. . . . + +"Uhm, Carlton," I babble, "I'm sure he's a really nice kid and all, +but that's a terrible temptation to set in front of somebody who's, +you know, into computers and just starting out. . . ." + +"Yeah," he says, "that did occur to me." For the first time I begin +to suspect that he's pulling my leg. + +He seems proudest when he shows me an ongoing project called JICC, +Joint Intelligence Control Council. It's based on the services provided +by EPIC, the El Paso Intelligence Center, which supplies data and intelligence +to the Drug Enforcement Administration, the Customs Service, the Coast Guard, +and the state police of the four southern border states. Certain EPIC files +can now be accessed by drug-enforcement police of Central America, +South America and the Caribbean, who can also trade information +among themselves. Using a telecom program called "White Hat," +written by two brothers named Lopez from the Dominican Republic, +police can now network internationally on inexpensive PCs. +Carlton Fitzpatrick is teaching a class of drug-war agents +from the Third World, and he's very proud of their progress. +Perhaps soon the sophisticated smuggling networks of the +Medellin Cartel will be matched by a sophisticated computer +network of the Medellin Cartel's sworn enemies. They'll track boats, +track contraband, track the international drug-lords who now leap over +borders with great ease, defeating the police through the clever use +of fragmented national jurisdictions. + +JICC and EPIC must remain beyond the scope of this book. +They seem to me to be very large topics fraught with complications +that I am not fit to judge. I do know, however, that the international, +computer-assisted networking of police, across national boundaries, +is something that Carlton Fitzpatrick considers very important, +a harbinger of a desirable future. I also know that networks +by their nature ignore physical boundaries. And I also know +that where you put communications you put a community, +and that when those communities become self-aware +they will fight to preserve themselves and to expand their influence. +I make no judgements whether this is good or bad. +It's just cyberspace; it's just the way things are. + +I asked Carlton Fitzpatrick what advice he would have for +a twenty-year-old who wanted to shine someday in the world +of electronic law enforcement. + +He told me that the number one rule was simply not to be +scared of computers. You don't need to be an obsessive +"computer weenie," but you mustn't be buffaloed just because +some machine looks fancy. The advantages computers give +smart crooks are matched by the advantages they give smart cops. +Cops in the future will have to enforce the law "with their heads, +not their holsters." Today you can make good cases without ever +leaving your office. In the future, cops who resist the computer +revolution will never get far beyond walking a beat. + +I asked Carlton Fitzpatrick if he had some single message for the public; +some single thing that he would most like the American public to know +about his work. + +He thought about it while. "Yes," he said finally. "TELL me the rules, +and I'll TEACH those rules!" He looked me straight in the eye. +"I do the best that I can." + + + +PART FOUR: THE CIVIL LIBERTARIANS + + +The story of the Hacker Crackdown, as we have followed it thus far, +has been technological, subcultural, criminal and legal. +The story of the Civil Libertarians, though it partakes +of all those other aspects, is profoundly and thoroughly POLITICAL. + +In 1990, the obscure, long-simmering struggle over the ownership +and nature of cyberspace became loudly and irretrievably public. +People from some of the oddest corners of American society suddenly +found themselves public figures. Some of these people found this +situation much more than they had ever bargained for. They backpedalled, +and tried to retreat back to the mandarin obscurity of their cozy +subcultural niches. This was generally to prove a mistake. + +But the civil libertarians seized the day in 1990. They found themselves +organizing, propagandizing, podium-pounding, persuading, touring, +negotiating, posing for publicity photos, submitting to interviews, +squinting in the limelight as they tried a tentative, but growingly +sophisticated, buck-and-wing upon the public stage. + +It's not hard to see why the civil libertarians should have +this competitive advantage. + +The hackers of the digital underground are an hermetic elite. +They find it hard to make any remotely convincing case for +their actions in front of the general public. Actually, +hackers roundly despise the "ignorant" public, and have never +trusted the judgement of "the system." Hackers do propagandize, +but only among themselves, mostly in giddy, badly spelled manifestos +of class warfare, youth rebellion or naive techie utopianism. +Hackers must strut and boast in order to establish and preserve +their underground reputations. But if they speak out too loudly +and publicly, they will break the fragile surface-tension of the underground, +and they will be harrassed or arrested. Over the longer term, +most hackers stumble, get busted, get betrayed, or simply give up. +As a political force, the digital underground is hamstrung. + +The telcos, for their part, are an ivory tower under protracted seige. +They have plenty of money with which to push their calculated public image, +but they waste much energy and goodwill attacking one another with +slanderous and demeaning ad campaigns. The telcos have suffered +at the hands of politicians, and, like hackers, they don't trust +the public's judgement. And this distrust may be well-founded. +Should the general public of the high-tech 1990s come to understand +its own best interests in telecommunications, that might well pose +a grave threat to the specialized technical power and authority +that the telcos have relished for over a century. The telcos do +have strong advantages: loyal employees, specialized expertise, +influence in the halls of power, tactical allies in law enforcement, +and unbelievably vast amounts of money. But politically speaking, they lack +genuine grassroots support; they simply don't seem to have many friends. + +Cops know a lot of things other people don't know. +But cops willingly reveal only those aspects of their +knowledge that they feel will meet their institutional +purposes and further public order. Cops have respect, +they have responsibilities, they have power in the streets +and even power in the home, but cops don't do particularly +well in limelight. When pressed, they will step out in the +public gaze to threaten bad-guys, or to cajole prominent citizens, +or perhaps to sternly lecture the naive and misguided. +But then they go back within their time-honored fortress +of the station-house, the courtroom and the rule-book. + +The electronic civil libertarians, however, have proven to be +born political animals. They seemed to grasp very early on +the postmodern truism that communication is power. Publicity is power. +Soundbites are power. The ability to shove one's issue onto the public +agenda--and KEEP IT THERE--is power. Fame is power. Simple personal +fluency and eloquence can be power, if you can somehow catch the +public's eye and ear. + +The civil libertarians had no monopoly on "technical power"-- +though they all owned computers, most were not particularly +advanced computer experts. They had a good deal of money, +but nowhere near the earthshaking wealth and the galaxy +of resources possessed by telcos or federal agencies. +They had no ability to arrest people. They carried +out no phreak and hacker covert dirty-tricks. + +But they really knew how to network. + +Unlike the other groups in this book, the civil libertarians +have operated very much in the open, more or less right +in the public hurly-burly. They have lectured audiences galore +and talked to countless journalists, and have learned to +refine their spiels. They've kept the cameras clicking, +kept those faxes humming, swapped that email, +run those photocopiers on overtime, licked envelopes +and spent small fortunes on airfare and long-distance. +In an information society, this open, overt, obvious activity +has proven to be a profound advantage. + +In 1990, the civil libertarians of cyberspace assembled +out of nowhere in particular, at warp speed. This "group" +(actually, a networking gaggle of interested parties +which scarcely deserves even that loose term) has almost nothing +in the way of formal organization. Those formal civil libertarian +organizations which did take an interest in cyberspace issues, +mainly the Computer Professionals for Social Responsibility +and the American Civil Liberties Union, were carried along +by events in 1990, and acted mostly as adjuncts, +underwriters or launching-pads. + +The civil libertarians nevertheless enjoyed the greatest success +of any of the groups in the Crackdown of 1990. At this writing, +their future looks rosy and the political initiative is firmly in their hands. +This should be kept in mind as we study the highly unlikely lives +and lifestyles of the people who actually made this happen. + +# + +In June 1989, Apple Computer, Inc., of Cupertino, +California, had a problem. Someone had illicitly copied +a small piece of Apple's proprietary software, software +which controlled an internal chip driving the Macintosh +screen display. This Color QuickDraw source code was +a closely guarded piece of Apple's intellectual property. +Only trusted Apple insiders were supposed to possess it. + +But the "NuPrometheus League" wanted things otherwise. +This person (or persons) made several illicit copies +of this source code, perhaps as many as two dozen. +He (or she, or they) then put those illicit floppy disks +into envelopes and mailed them to people all over America: +people in the computer industry who were associated with, +but not directly employed by, Apple Computer. + +The NuPrometheus caper was a complex, highly ideological, +and very hacker-like crime. Prometheus, it will be recalled, +stole the fire of the Gods and gave this potent gift to the +general ranks of downtrodden mankind. A similar god-in-the-manger +attitude was implied for the corporate elite of Apple Computer, +while the "Nu" Prometheus had himself cast in the role of rebel demigod. +The illicitly copied data was given away for free. + +The new Prometheus, whoever he was, escaped the +fate of the ancient Greek Prometheus, who was chained +to a rock for centuries by the vengeful gods while an eagle +tore and ate his liver. On the other hand, NuPrometheus +chickened out somewhat by comparison with his role model. +The small chunk of Color QuickDraw code he had filched +and replicated was more or less useless to Apple's +industrial rivals (or, in fact, to anyone else). +Instead of giving fire to mankind, it was more as if +NuPrometheus had photocopied the schematics for part of a Bic lighter. +The act was not a genuine work of industrial espionage. +It was best interpreted as a symbolic, deliberate slap +in the face for the Apple corporate heirarchy. + +Apple's internal struggles were well-known in the industry. Apple's founders, +Jobs and Wozniak, had both taken their leave long since. Their raucous core +of senior employees had been a barnstorming crew of 1960s Californians, +many of them markedly less than happy with the new button-down multimillion +dollar regime at Apple. Many of the programmers and developers who had +invented the Macintosh model in the early 1980s had also taken their leave of +the company. It was they, not the current masters of Apple's corporate fate, +who had invented the stolen Color QuickDraw code. The NuPrometheus stunt +was well-calculated to wound company morale. + +Apple called the FBI. The Bureau takes an interest in high-profile +intellectual-property theft cases, industrial espionage and theft +of trade secrets. These were likely the right people to call, +and rumor has it that the entities responsible were in fact discovered +by the FBI, and then quietly squelched by Apple management. NuPrometheus +was never publicly charged with a crime, or prosecuted, or jailed. +But there were no further illicit releases of Macintosh internal software. +Eventually the painful issue of NuPrometheus was allowed to fade. + +In the meantime, however, a large number of puzzled bystanders +found themselves entertaining surprise guests from the FBI. + +One of these people was John Perry Barlow. Barlow is a most unusual man, +difficult to describe in conventional terms. He is perhaps best known as +a songwriter for the Grateful Dead, for he composed lyrics for +"Hell in a Bucket," "Picasso Moon," "Mexicali Blues," "I Need a Miracle," +and many more; he has been writing for the band since 1970. + +Before we tackle the vexing question as to why a rock lyricist +should be interviewed by the FBI in a computer-crime case, +it might be well to say a word or two about the Grateful Dead. +The Grateful Dead are perhaps the most successful and long-lasting +of the numerous cultural emanations from the Haight-Ashbury district +of San Francisco, in the glory days of Movement politics and +lysergic transcendance. The Grateful Dead are a nexus, a veritable +whirlwind, of applique decals, psychedelic vans, tie-dyed T-shirts, +earth-color denim, frenzied dancing and open and unashamed drug use. +The symbols, and the realities, of Californian freak power surround +the Grateful Dead like knotted macrame. + +The Grateful Dead and their thousands of Deadhead devotees +are radical Bohemians. This much is widely understood. +Exactly what this implies in the 1990s is rather more problematic. + +The Grateful Dead are among the world's most popular +and wealthy entertainers: number 20, according to Forbes magazine, +right between M.C. Hammer and Sean Connery. In 1990, this jeans-clad +group of purported raffish outcasts earned seventeen million dollars. +They have been earning sums much along this line for quite some time now. + +And while the Dead are not investment bankers or three-piece-suit +tax specialists--they are, in point of fact, hippie musicians-- +this money has not been squandered in senseless Bohemian excess. +The Dead have been quietly active for many years, funding various +worthy activities in their extensive and widespread cultural community. + +The Grateful Dead are not conventional players in the American +power establishment. They nevertheless are something of a force +to be reckoned with. They have a lot of money and a lot of friends +in many places, both likely and unlikely. + +The Dead may be known for back-to-the-earth environmentalist rhetoric, +but this hardly makes them anti-technological Luddites. On the contrary, +like most rock musicians, the Grateful Dead have spent their entire adult +lives in the company of complex electronic equipment. They have funds to burn +on any sophisticated tool and toy that might happen to catch their fancy. +And their fancy is quite extensive. + +The Deadhead community boasts any number of recording engineers, +lighting experts, rock video mavens, electronic technicians +of all descriptions. And the drift goes both ways. Steve Wozniak, +Apple's co-founder, used to throw rock festivals. Silicon Valley rocks out. + +These are the 1990s, not the 1960s. Today, for a surprising number of people +all over America, the supposed dividing line between Bohemian and technician +simply no longer exists. People of this sort may have a set of windchimes +and a dog with a knotted kerchief 'round its neck, but they're also quite +likely to own a multimegabyte Macintosh running MIDI synthesizer software +and trippy fractal simulations. These days, even Timothy Leary himself, +prophet of LSD, does virtual-reality computer-graphics demos in +his lecture tours. + +John Perry Barlow is not a member of the Grateful Dead. He is, however, +a ranking Deadhead. + +Barlow describes himself as a "techno-crank." A vague term like +"social activist" might not be far from the mark, either. +But Barlow might be better described as a "poet"--if one keeps in mind +Percy Shelley's archaic definition of poets as "unacknowledged legislators +of the world." + +Barlow once made a stab at acknowledged legislator status. In 1987, +he narrowly missed the Republican nomination for a seat in the +Wyoming State Senate. Barlow is a Wyoming native, the third-generation +scion of a well-to-do cattle-ranching family. He is in his early forties, +married and the father of three daughters. + +Barlow is not much troubled by other people's narrow notions of consistency. +In the late 1980s, this Republican rock lyricist cattle rancher sold his ranch +and became a computer telecommunications devotee. + +The free-spirited Barlow made this transition with ease. He genuinely +enjoyed computers. With a beep of his modem, he leapt from small-town +Pinedale, Wyoming, into electronic contact with a large and lively crowd +of bright, inventive, technological sophisticates from all over the world. +Barlow found the social milieu of computing attractive: its fast-lane pace, +its blue-sky rhetoric, its open-endedness. Barlow began dabbling in +computer journalism, with marked success, as he was a quick study, +and both shrewd and eloquent. He frequently travelled to San Francisco +to network with Deadhead friends. There Barlow made extensive contacts +throughout the Californian computer community, including friendships +among the wilder spirits at Apple. + +In May 1990, Barlow received a visit from a local Wyoming agent of the FBI. +The NuPrometheus case had reached Wyoming. + +Barlow was troubled to find himself under investigation in an +area of his interests once quite free of federal attention. +He had to struggle to explain the very nature of computer-crime +to a headscratching local FBI man who specialized in cattle-rustling. +Barlow, chatting helpfully and demonstrating the wonders of his modem +to the puzzled fed, was alarmed to find all "hackers" generally under +FBI suspicion as an evil influence in the electronic community. +The FBI, in pursuit of a hacker called "NuPrometheus," were tracing +attendees of a suspect group called the Hackers Conference. + +The Hackers Conference, which had been started in 1984, was a +yearly Californian meeting of digital pioneers and enthusiasts. +The hackers of the Hackers Conference had little if anything to do +with the hackers of the digital underground. On the contrary, +the hackers of this conference were mostly well-to-do Californian +high-tech CEOs, consultants, journalists and entrepreneurs. +(This group of hackers were the exact sort of "hackers" +most likely to react with militant fury at any criminal +degradation of the term "hacker.") + +Barlow, though he was not arrested or accused of a crime, +and though his computer had certainly not gone out the door, +was very troubled by this anomaly. He carried the word to the Well. + +Like the Hackers Conference, "the Well" was an emanation of the +Point Foundation. Point Foundation, the inspiration of a wealthy +Californian 60s radical named Stewart Brand, was to be a major +launch-pad of the civil libertarian effort. + +Point Foundation's cultural efforts, like those of their fellow Bay Area +Californians the Grateful Dead, were multifaceted and multitudinous. +Rigid ideological consistency had never been a strong suit of the +Whole Earth Catalog. This Point publication had enjoyed a strong +vogue during the late 60s and early 70s, when it offered hundreds +of practical (and not so practical) tips on communitarian living, +environmentalism, and getting back-to-the-land. The Whole Earth Catalog, +and its sequels, sold two and half million copies and won a +National Book Award. + +With the slow collapse of American radical dissent, the Whole Earth Catalog +had slipped to a more modest corner of the cultural radar; but in its +magazine incarnation, CoEvolution Quarterly, the Point Foundation +continued to offer a magpie potpourri of "access to tools and ideas." + +CoEvolution Quarterly, which started in 1974, was never a widely +popular magazine. Despite periodic outbreaks of millenarian fervor, +CoEvolution Quarterly failed to revolutionize Western civilization +and replace leaden centuries of history with bright new Californian paradigms. +Instead, this propaganda arm of Point Foundation cakewalked a fine line between +impressive brilliance and New Age flakiness. CoEvolution Quarterly carried +no advertising, cost a lot, and came out on cheap newsprint with modest +black-and-white graphics. It was poorly distributed, and spread mostly +by subscription and word of mouth. + +It could not seem to grow beyond 30,000 subscribers. +And yet--it never seemed to shrink much, either. +Year in, year out, decade in, decade out, some strange +demographic minority accreted to support the magazine. +The enthusiastic readership did not seem to have much +in the way of coherent politics or ideals. It was sometimes +hard to understand what held them together (if the often bitter +debate in the letter-columns could be described as "togetherness"). + +But if the magazine did not flourish, it was resilient; it got by. +Then, in 1984, the birth-year of the Macintosh computer, +CoEvolution Quarterly suddenly hit the rapids. Point Foundation +had discovered the computer revolution. Out came the Whole Earth +Software Catalog of 1984, arousing headscratching doubts among +the tie-dyed faithful, and rabid enthusiasm among the nascent +"cyberpunk" milieu, present company included. Point Foundation +started its yearly Hackers Conference, and began to take an +extensive interest in the strange new possibilities of +digital counterculture. CoEvolution Quarterlyfolded its teepee, +replaced by Whole Earth Software Review and eventually by Whole Earth +Review (the magazine's present incarnation, currently under +the editorship of virtual-reality maven Howard Rheingold). + +1985 saw the birth of the "WELL"--the "Whole Earth 'Lectronic Link." +The Well was Point Foundation's bulletin board system. + +As boards went, the Well was an anomaly from the beginning, +and remained one. It was local to San Francisco. +It was huge, with multiple phonelines and enormous files +of commentary. Its complex UNIX-based software might be +most charitably described as "user-opaque." It was run on +a mainframe out of the rambling offices of a non-profit +cultural foundation in Sausalito. And it was crammed with +fans of the Grateful Dead. + +Though the Well was peopled by chattering hipsters of the Bay Area +counterculture, it was by no means a "digital underground" board. +Teenagers were fairly scarce; most Well users (known as "Wellbeings") +were thirty- and forty-something Baby Boomers. They tended to work +in the information industry: hardware, software, telecommunications, +media, entertainment. Librarians, academics, and journalists were +especially common on the Well, attracted by Point Foundation's +open-handed distribution of "tools and ideas." + +There were no anarchy files on the Well, scarcely a +dropped hint about access codes or credit-card theft. +No one used handles. Vicious "flame-wars" were held to +a comparatively civilized rumble. Debates were sometimes sharp, +but no Wellbeing ever claimed that a rival had disconnected his phone, +trashed his house, or posted his credit card numbers. + +The Well grew slowly as the 1980s advanced. It charged a modest sum +for access and storage, and lost money for years--but not enough to hamper +the Point Foundation, which was nonprofit anyway. By 1990, the Well +had about five thousand users. These users wandered about a gigantic +cyberspace smorgasbord of "Conferences", each conference itself consisting +of a welter of "topics," each topic containing dozens, sometimes hundreds +of comments, in a tumbling, multiperson debate that could last for months +or years on end. + + +In 1991, the Well's list of conferences looked like this: + + +CONFERENCES ON THE WELL + +WELL "Screenzine" Digest (g zine) + +Best of the WELL - vintage material - (g best) + +Index listing of new topics in all conferences - (g newtops) + +Business - Education +---------------------- + +Apple Library Users Group(g alug) Agriculture (g agri) +Brainstorming (g brain) Classifieds (g cla) +Computer Journalism (g cj) Consultants (g consult) +Consumers (g cons) Design (g design) +Desktop Publishing (g desk) Disability (g disability) +Education (g ed) Energy (g energy91) +Entrepreneurs (g entre) Homeowners (g home) +Indexing (g indexing) Investments (g invest) +Kids91 (g kids) Legal (g legal) +One Person Business (g one) +Periodical/newsletter (g per) +Telecomm Law (g tcl) The Future (g fut) +Translators (g trans) Travel (g tra) +Work (g work) + +Electronic Frontier Foundation (g eff) +Computers, Freedom & Privacy (g cfp) +Computer Professionals for Social Responsibility (g cpsr) + +Social - Political - Humanities +--------------------------------- + +Aging (g gray) AIDS (g aids) +Amnesty International (g amnesty) Archives (g arc) +Berkeley (g berk) Buddhist (g wonderland) +Christian (g cross) Couples (g couples) +Current Events (g curr) Dreams (g dream) +Drugs (g dru) East Coast (g east) +Emotional Health@@@@ (g private) Erotica (g eros) +Environment (g env) Firearms (g firearms) +First Amendment (g first) Fringes of Reason (g fringes) +Gay (g gay) Gay (Private)# (g gaypriv) +Geography (g geo) German (g german) +Gulf War (g gulf) Hawaii (g aloha) +Health (g heal) History (g hist) +Holistic (g holi) Interview (g inter) +Italian (g ital) Jewish (g jew) +Liberty (g liberty) Mind (g mind) +Miscellaneous (g misc) Men on the WELL@@ (g mow) +Network Integration (g origin) Nonprofits (g non) +North Bay (g north) Northwest (g nw) +Pacific Rim (g pacrim) Parenting (g par) +Peace (g pea) Peninsula (g pen) +Poetry (g poetry) Philosophy (g phi) +Politics (g pol) Psychology (g psy) +Psychotherapy (g therapy) Recovery## (g recovery) +San Francisco (g sanfran) Scams (g scam) +Sexuality (g sex) Singles (g singles) +Southern (g south) Spanish (g spanish) +Spirituality (g spirit) Tibet (g tibet) +Transportation (g transport) True Confessions (g tru) +Unclear (g unclear) WELL Writer's Workshop@@@(g www) +Whole Earth (g we) Women on the WELL@(g wow) +Words (g words) Writers (g wri) + +@@@@Private Conference - mail wooly for entry +@@@Private conference - mail sonia for entry +@@Private conference - mail flash for entry +@ Private conference - mail reva for entry +# Private Conference - mail hudu for entry +## Private Conference - mail dhawk for entry + +Arts - Recreation - Entertainment +----------------------------------- +ArtCom Electronic Net (g acen) +Audio-Videophilia (g aud) +Bicycles (g bike) Bay Area Tonight@@(g bat) +Boating (g wet) Books (g books) +CD's (g cd) Comics (g comics) +Cooking (g cook) Flying (g flying) +Fun (g fun) Games (g games) +Gardening (g gard) Kids (g kids) +Nightowls@ (g owl) Jokes (g jokes) +MIDI (g midi) Movies (g movies) +Motorcycling (g ride) Motoring (g car) +Music (g mus) On Stage (g onstage) +Pets (g pets) Radio (g rad) +Restaurant (g rest) Science Fiction (g sf) +Sports (g spo) Star Trek (g trek) +Television (g tv) Theater (g theater) +Weird (g weird) Zines/Factsheet Five(g f5) +@Open from midnight to 6am +@@Updated daily + +Grateful Dead +------------- +Grateful Dead (g gd) Deadplan@ (g dp) +Deadlit (g deadlit) Feedback (g feedback) +GD Hour (g gdh) Tapes (g tapes) +Tickets (g tix) Tours (g tours) + +@Private conference - mail tnf for entry + +Computers +----------- +AI/Forth/Realtime (g realtime) Amiga (g amiga) +Apple (g app) Computer Books (g cbook) +Art & Graphics (g gra) Hacking (g hack) +HyperCard (g hype) IBM PC (g ibm) +LANs (g lan) Laptop (g lap) +Macintosh (g mac) Mactech (g mactech) +Microtimes (g microx) Muchomedia (g mucho) +NeXt (g next) OS/2 (g os2) +Printers (g print) Programmer's Net (g net) +Siggraph (g siggraph) Software Design (g sdc) +Software/Programming (g software) +Software Support (g ssc) +Unix (g unix) Windows (g windows) +Word Processing (g word) + +Technical - Communications +---------------------------- +Bioinfo (g bioinfo) Info (g boing) +Media (g media) NAPLPS (g naplps) +Netweaver (g netweaver) Networld (g networld) +Packet Radio (g packet) Photography (g pho) +Radio (g rad) Science (g science) +Technical Writers (g tec) Telecommunications(g tele) +Usenet (g usenet) Video (g vid) +Virtual Reality (g vr) + +The WELL Itself +--------------- +Deeper (g deeper) Entry (g ent) +General (g gentech) Help (g help) +Hosts (g hosts) Policy (g policy) +System News (g news) Test (g test) + +The list itself is dazzling, bringing to the untutored eye +a dizzying impression of a bizarre milieu of mountain-climbing +Hawaiian holistic photographers trading true-life confessions +with bisexual word-processing Tibetans. + +But this confusion is more apparent than real. Each of these conferences +was a little cyberspace world in itself, comprising dozens and perhaps +hundreds of sub-topics. Each conference was commonly frequented by +a fairly small, fairly like-minded community of perhaps a few dozen people. +It was humanly impossible to encompass the entire Well (especially since +access to the Well's mainframe computer was billed by the hour). +Most long-time users contented themselves with a few favorite +topical neighborhoods, with the occasional foray elsewhere +for a taste of exotica. But especially important news items, +and hot topical debates, could catch the attention of the entire +Well community. + +Like any community, the Well had its celebrities, and John Perry Barlow, +the silver-tongued and silver-modemed lyricist of the Grateful Dead, +ranked prominently among them. It was here on the Well that Barlow +posted his true-life tale of computer-crime encounter with the FBI. + +The story, as might be expected, created a great stir. The Well was +already primed for hacker controversy. In December 1989, Harper's magazine +had hosted a debate on the Well about the ethics of illicit computer intrusion. +While over forty various computer-mavens took part, Barlow proved a star +in the debate. So did "Acid Phreak" and "Phiber Optik," a pair of young +New York hacker-phreaks whose skills at telco switching-station intrusion +were matched only by their apparently limitless hunger for fame. +The advent of these two boldly swaggering outlaws in the precincts +of the Well created a sensation akin to that of Black Panthers +at a cocktail party for the radically chic. + +Phiber Optik in particular was to seize the day in 1990. +A devotee of the 2600 circle and stalwart of the New York +hackers' group "Masters of Deception," Phiber Optik was +a splendid exemplar of the computer intruder as committed dissident. +The eighteen-year-old Optik, a high-school dropout and part-time +computer repairman, was young, smart, and ruthlessly obsessive, +a sharp-dressing, sharp-talking digital dude who was utterly +and airily contemptuous of anyone's rules but his own. +By late 1991, Phiber Optik had appeared in Harper's, +Esquire, The New York Times, in countless public debates +and conventions, even on a television show hosted by Geraldo Rivera. + +Treated with gingerly respect by Barlow and other Well mavens, +Phiber Optik swiftly became a Well celebrity. Strangely, despite +his thorny attitude and utter single-mindedness, Phiber Optik seemed +to arouse strong protective instincts in most of the people who met him. +He was great copy for journalists, always fearlessly ready to swagger, +and, better yet, to actually DEMONSTRATE some off-the-wall digital stunt. +He was a born media darling. + +Even cops seemed to recognize that there was something peculiarly unworldly +and uncriminal about this particular troublemaker. He was so bold, +so flagrant, so young, and so obviously doomed, that even those +who strongly disapproved of his actions grew anxious for his welfare, +and began to flutter about him as if he were an endangered seal pup. + +In January 24, 1990 (nine days after the Martin Luther King Day Crash), +Phiber Optik, Acid Phreak, and a third NYC scofflaw named Scorpion were +raided by the Secret Service. Their computers went out the door, +along with the usual blizzard of papers, notebooks, compact disks, +answering machines, Sony Walkmans, etc. Both Acid Phreak and +Phiber Optik were accused of having caused the Crash. + +The mills of justice ground slowly. The case eventually fell into +the hands of the New York State Police. Phiber had lost his machinery +in the raid, but there were no charges filed against him for over a year. +His predicament was extensively publicized on the Well, where it caused +much resentment for police tactics. It's one thing to merely hear about +a hacker raided or busted; it's another to see the police attacking someone +you've come to know personally, and who has explained his motives at length. +Through the Harper's debate on the Well, it had become clear to the +Wellbeings that Phiber Optik was not in fact going to "hurt anything." +In their own salad days, many Wellbeings had tasted tear-gas in pitched +street-battles with police. They were inclined to indulgence for +acts of civil disobedience. + +Wellbeings were also startled to learn of the draconian thoroughness +of a typical hacker search-and-seizure. It took no great stretch of +imagination for them to envision themselves suffering much the same treatment. + +As early as January 1990, sentiment on the Well had already begun to sour, +and people had begun to grumble that "hackers" were getting a raw deal +from the ham-handed powers-that-be. The resultant issue of Harper's +magazine posed the question as to whether computer-intrusion was a "crime" +at all. As Barlow put it later: "I've begun to wonder if we wouldn't +also regard spelunkers as desperate criminals if AT&T owned all the caves." + +In February 1991, more than a year after the raid on his home, +Phiber Optik was finally arrested, and was charged with first-degree +Computer Tampering and Computer Trespass, New York state offenses. +He was also charged with a theft-of-service misdemeanor, involving a complex +free-call scam to a 900 number. Phiber Optik pled guilty to the misdemeanor +charge, and was sentenced to 35 hours of community service. + +This passing harassment from the unfathomable world of straight people +seemed to bother Optik himself little if at all. Deprived of his computer +by the January search-and-seizure, he simply bought himself a portable +computer so the cops could no longer monitor the phone where he lived +with his Mom, and he went right on with his depredations, sometimes on +live radio or in front of television cameras. + +The crackdown raid may have done little to dissuade Phiber Optik, +but its galling affect on the Wellbeings was profound. As 1990 rolled on, +the slings and arrows mounted: the Knight Lightning raid, +the Steve Jackson raid, the nation-spanning Operation Sundevil. +The rhetoric of law enforcement made it clear that there was, +in fact, a concerted crackdown on hackers in progress. + +The hackers of the Hackers Conference, the Wellbeings, and their ilk, +did not really mind the occasional public misapprehension of "hacking;" +if anything, this membrane of differentiation from straight society +made the "computer community" feel different, smarter, better. +They had never before been confronted, however, by a concerted +vilification campaign. + +Barlow's central role in the counter-struggle was one of the major +anomalies of 1990. Journalists investigating the controversy +often stumbled over the truth about Barlow, but they commonly +dusted themselves off and hurried on as if nothing had happened. +It was as if it were TOO MUCH TO BELIEVE that a 1960s freak +from the Grateful Dead had taken on a federal law enforcement operation +head-to-head and ACTUALLY SEEMED TO BE WINNING! + +Barlow had no easily detectable power-base for a political struggle +of this kind. He had no formal legal or technical credentials. +Barlow was, however, a computer networker of truly stellar brilliance. +He had a poet's gift of concise, colorful phrasing. He also had a +journalist's shrewdness, an off-the-wall, self-deprecating wit, +and a phenomenal wealth of simple personal charm. + +The kind of influence Barlow possessed is fairly common currency +in literary, artistic, or musical circles. A gifted critic can +wield great artistic influence simply through defining +the temper of the times, by coining the catch-phrases +and the terms of debate that become the common currency of the period. +(And as it happened, Barlow WAS a part-time art critic, +with a special fondness for the Western art of Frederic Remington.) + +Barlow was the first commentator to adopt William Gibson's +striking science-fictional term "cyberspace" as a synonym +for the present-day nexus of computer and telecommunications networks. +Barlow was insistent that cyberspace should be regarded as +a qualitatively new world, a "frontier." According to Barlow, +the world of electronic communications, now made visible through +the computer screen, could no longer be usefully regarded +as just a tangle of high-tech wiring. Instead, it had become +a PLACE, cyberspace, which demanded a new set of metaphors, +a new set of rules and behaviors. The term, as Barlow employed it, +struck a useful chord, and this concept of cyberspace was picked up +by Time, Scientific American, computer police, hackers, and even +Constitutional scholars. "Cyberspace" now seems likely to become +a permanent fixture of the language. + +Barlow was very striking in person: a tall, craggy-faced, bearded, +deep-voiced Wyomingan in a dashing Western ensemble of jeans, jacket, +cowboy boots, a knotted throat-kerchief and an ever-present Grateful Dead +cloisonne lapel pin. + +Armed with a modem, however, Barlow was truly in his element. +Formal hierarchies were not Barlow's strong suit; he rarely missed +a chance to belittle the "large organizations and their drones," +with their uptight, institutional mindset. Barlow was very much +of the free-spirit persuasion, deeply unimpressed by brass-hats +and jacks-in-office. But when it came to the digital grapevine, +Barlow was a cyberspace ad-hocrat par excellence. + +There was not a mighty army of Barlows. There was only one Barlow, +and he was a fairly anomolous individual. However, the situation only +seemed to REQUIRE a single Barlow. In fact, after 1990, many people +must have concluded that a single Barlow was far more than +they'd ever bargained for. + +Barlow's querulous mini-essay about his encounter with the FBI +struck a strong chord on the Well. A number of other free spirits +on the fringes of Apple Computing had come under suspicion, +and they liked it not one whit better than he did. + +One of these was Mitchell Kapor, the co-inventor of the spreadsheet +program "Lotus 1-2-3" and the founder of Lotus Development Corporation. +Kapor had written-off the passing indignity of being fingerprinted +down at his own local Boston FBI headquarters, but Barlow's post +made the full national scope of the FBI's dragnet clear to Kapor. +The issue now had Kapor's full attention. As the Secret Service +swung into anti-hacker operation nationwide in 1990, Kapor watched +every move with deep skepticism and growing alarm. + +As it happened, Kapor had already met Barlow, who had interviewed Kapor +for a California computer journal. Like most people who met Barlow, +Kapor had been very taken with him. Now Kapor took it upon himself +to drop in on Barlow for a heart-to-heart talk about the situation. + +Kapor was a regular on the Well. Kapor had been a devotee of the +Whole Earth Catalogsince the beginning, and treasured a complete run +of the magazine. And Kapor not only had a modem, but a private jet. +In pursuit of the scattered high-tech investments of Kapor Enterprises Inc., +his personal, multi-million dollar holding company, Kapor commonly crossed +state lines with about as much thought as one might give to faxing a letter. + +The Kapor-Barlow council of June 1990, in Pinedale, Wyoming, was the start +of the Electronic Frontier Foundation. Barlow swiftly wrote a manifesto, +"Crime and Puzzlement," which announced his, and Kapor's, intention +to form a political organization to "raise and disburse funds for education, +lobbying, and litigation in the areas relating to digital speech and the +extension of the Constitution into Cyberspace." + +Furthermore, proclaimed the manifesto, the foundation would +"fund, conduct, and support legal efforts to demonstrate +that the Secret Service has exercised prior restraint on publications, +limited free speech, conducted improper seizure of equipment and data, +used undue force, and generally conducted itself in a fashion which +is arbitrary, oppressive, and unconstitutional." + +"Crime and Puzzlement" was distributed far and wide through computer +networking channels, and also printed in the Whole Earth Review. +The sudden declaration of a coherent, politicized counter-strike +from the ranks of hackerdom electrified the community. Steve Wozniak +(perhaps a bit stung by the NuPrometheus scandal) swiftly offered +to match any funds Kapor offered the Foundation. + +John Gilmore, one of the pioneers of Sun Microsystems, immediately offered +his own extensive financial and personal support. Gilmore, an ardent +libertarian, was to prove an eloquent advocate of electronic privacy issues, +especially freedom from governmental and corporate computer-assisted +surveillance of private citizens. + +A second meeting in San Francisco rounded up further allies: +Stewart Brand of the Point Foundation, virtual-reality pioneers +Jaron Lanier and Chuck Blanchard, network entrepreneur and venture +capitalist Nat Goldhaber. At this dinner meeting, the activists settled on +a formal title: the Electronic Frontier Foundation, Incorporated. +Kapor became its president. A new EFF Conference was opened on +the Point Foundation's Well, and the Well was declared +"the home of the Electronic Frontier Foundation." + +Press coverage was immediate and intense. Like their +nineteenth-century spiritual ancestors, Alexander Graham Bell +and Thomas Watson, the high-tech computer entrepreneurs +of the 1970s and 1980s--people such as Wozniak, Jobs, Kapor, +Gates, and H. Ross Perot, who had raised themselves by their bootstraps +to dominate a glittering new industry--had always made very good copy. + +But while the Wellbeings rejoiced, the press in general seemed +nonplussed by the self-declared "civilizers of cyberspace." +EFF's insistence that the war against "hackers" involved grave +Constitutional civil liberties issues seemed somewhat farfetched, +especially since none of EFF's organizers were lawyers +or established politicians. The business press in particular +found it easier to seize on the apparent core of the story-- +that high-tech entrepreneur Mitchell Kapor had established +a "defense fund for hackers." Was EFF a genuinely important +political development--or merely a clique of wealthy eccentrics, +dabbling in matters better left to the proper authorities? +The jury was still out. + +But the stage was now set for open confrontation. +And the first and the most critical battle was the +hacker show-trial of "Knight Lightning." + +# + +It has been my practice throughout this book to refer to hackers +only by their "handles." There is little to gain by giving +the real names of these people, many of whom are juveniles, +many of whom have never been convicted of any crime, and many +of whom had unsuspecting parents who have already suffered enough. + +But the trial of Knight Lightning on July 24-27, 1990, +made this particular "hacker" a nationally known public figure. +It can do no particular harm to himself or his family if I repeat +the long-established fact that his name is Craig Neidorf (pronounced NYE-dorf). + +Neidorf's jury trial took place in the United States District Court, +Northern District of Illinois, Eastern Division, with the +Honorable Nicholas J. Bua presiding. The United States of America +was the plaintiff, the defendant Mr. Neidorf. The defendant's attorney +was Sheldon T. Zenner of the Chicago firm of Katten, Muchin and Zavis. + +The prosecution was led by the stalwarts of the Chicago Computer Fraud +and Abuse Task Force: William J. Cook, Colleen D. Coughlin, and +David A. Glockner, all Assistant United States Attorneys. +The Secret Service Case Agent was Timothy M. Foley. + +It will be recalled that Neidorf was the co-editor of an underground hacker +"magazine" called Phrack. Phrack was an entirely electronic publication, +distributed through bulletin boards and over electronic networks. +It was amateur publication given away for free. Neidorf had never made +any money for his work in Phrack. Neither had his unindicted co-editor +"Taran King" or any of the numerous Phrack contributors. + +The Chicago Computer Fraud and Abuse Task Force, however, +had decided to prosecute Neidorf as a fraudster. +To formally admit that Phrack was a "magazine" +and Neidorf a "publisher" was to open a prosecutorial +Pandora's Box of First Amendment issues. To do this +was to play into the hands of Zenner and his EFF advisers, +which now included a phalanx of prominent New York civil rights +lawyers as well as the formidable legal staff of Katten, Muchin and Zavis. +Instead, the prosecution relied heavily on the issue of access device fraud: +Section 1029 of Title 18, the section from which the Secret Service drew +its most direct jurisdiction over computer crime. + +Neidorf's alleged crimes centered around the E911 Document. +He was accused of having entered into a fraudulent scheme with the Prophet, +who, it will be recalled, was the Atlanta LoD member who had illicitly +copied the E911 Document from the BellSouth AIMSX system. + +The Prophet himself was also a co-defendant in the Neidorf case, +part-and-parcel of the alleged "fraud scheme" to "steal" BellSouth's +E911 Document (and to pass the Document across state lines, +which helped establish the Neidorf trial as a federal case). +The Prophet, in the spirit of full co-operation, had agreed +to testify against Neidorf. + +In fact, all three of the Atlanta crew stood ready to testify against Neidorf. +Their own federal prosecutors in Atlanta had charged the Atlanta Three with: +(a) conspiracy, (b) computer fraud, (c) wire fraud, (d) access device fraud, +and (e) interstate transportation of stolen property (Title 18, Sections 371, +1030, 1343, 1029, and 2314). + +Faced with this blizzard of trouble, Prophet and Leftist had ducked +any public trial and had pled guilty to reduced charges--one conspiracy +count apiece. Urvile had pled guilty to that odd bit of Section 1029 +which makes it illegal to possess "fifteen or more" illegal access devices +(in his case, computer passwords). And their sentences were scheduled +for September 14, 1990--well after the Neidorf trial. As witnesses, +they could presumably be relied upon to behave. + +Neidorf, however, was pleading innocent. Most everyone else caught up +in the crackdown had "cooperated fully" and pled guilty in hope +of reduced sentences. (Steve Jackson was a notable exception, +of course, and had strongly protested his innocence from the +very beginning. But Steve Jackson could not get a day in court-- +Steve Jackson had never been charged with any crime in the first place.) + +Neidorf had been urged to plead guilty. But Neidorf was a political science +major and was disinclined to go to jail for "fraud" when he had not made +any money, had not broken into any computer, and had been publishing +a magazine that he considered protected under the First Amendment. + +Neidorf's trial was the ONLY legal action of the entire Crackdown +that actually involved bringing the issues at hand out for a public test +in front of a jury of American citizens. + +Neidorf, too, had cooperated with investigators. He had voluntarily +handed over much of the evidence that had led to his own indictment. +He had already admitted in writing that he knew that the E911 Document +had been stolen before he had "published" it in Phrack--or, from the +prosecution's point of view, illegally transported stolen property by wire +in something purporting to be a "publication." + +But even if the "publication" of the E911 Document was not held to be a crime, +that wouldn't let Neidorf off the hook. Neidorf had still received +the E911 Document when Prophet had transferred it to him from Rich Andrews' +Jolnet node. On that occasion, it certainly hadn't been "published"-- +it was hacker booty, pure and simple, transported across state lines. + +The Chicago Task Force led a Chicago grand jury to indict Neidorf +on a set of charges that could have put him in jail for thirty years. +When some of these charges were successfully challenged before Neidorf +actually went to trial, the Chicago Task Force rearranged his +indictment so that he faced a possible jail term of over sixty years! +As a first offender, it was very unlikely that Neidorf would in fact +receive a sentence so drastic; but the Chicago Task Force clearly +intended to see Neidorf put in prison, and his conspiratorial "magazine" +put permanently out of commission. This was a federal case, and Neidorf +was charged with the fraudulent theft of property worth almost +eighty thousand dollars. + +William Cook was a strong believer in high-profile prosecutions +with symbolic overtones. He often published articles on his work +in the security trade press, arguing that "a clear message had +to be sent to the public at large and the computer community +in particular that unauthorized attacks on computers and the theft +of computerized information would not be tolerated by the courts." + +The issues were complex, the prosecution's tactics somewhat unorthodox, +but the Chicago Task Force had proved sure-footed to date. "Shadowhawk" +had been bagged on the wing in 1989 by the Task Force, and sentenced +to nine months in prison, and a $10,000 fine. The Shadowhawk case involved +charges under Section 1030, the "federal interest computer" section. + +Shadowhawk had not in fact been a devotee of "federal-interest" computers +per se. On the contrary, Shadowhawk, who owned an AT&T home computer, +seemed to cherish a special aggression toward AT&T. He had bragged on +the underground boards "Phreak Klass 2600" and "Dr. Ripco" of his skills +at raiding AT&T, and of his intention to crash AT&T's national phone system. +Shadowhawk's brags were noticed by Henry Kluepfel of Bellcore Security, +scourge of the outlaw boards, whose relations with the Chicago Task Force +were long and intimate. + +The Task Force successfully established that Section 1030 applied to +the teenage Shadowhawk, despite the objections of his defense attorney. +Shadowhawk had entered a computer "owned" by U.S. Missile Command +and merely "managed" by AT&T. He had also entered an AT&T computer +located at Robbins Air Force Base in Georgia. Attacking AT&T was +of "federal interest" whether Shadowhawk had intended it or not. + +The Task Force also convinced the court that a piece of AT&T +software that Shadowhawk had illicitly copied from Bell Labs, +the "Artificial Intelligence C5 Expert System," was worth a cool +one million dollars. Shadowhawk's attorney had argued that +Shadowhawk had not sold the program and had made no profit from +the illicit copying. And in point of fact, the C5 Expert System +was experimental software, and had no established market value +because it had never been on the market in the first place. +AT&T's own assessment of a "one million dollar" figure for its +own intangible property was accepted without challenge +by the court, however. And the court concurred with +the government prosecutors that Shadowhawk showed clear +"intent to defraud" whether he'd gotten any money or not. +Shadowhawk went to jail. + +The Task Force's other best-known triumph had been the conviction +and jailing of "Kyrie." Kyrie, a true denizen of the digital +criminal underground, was a 36-year-old Canadian woman, +convicted and jailed for telecommunications fraud in Canada. +After her release from prison, she had fled the wrath of Canada Bell +and the Royal Canadian Mounted Police, and eventually settled, +very unwisely, in Chicago. + +"Kyrie," who also called herself "Long Distance Information," +specialized in voice-mail abuse. She assembled large numbers +of hot long-distance codes, then read them aloud into a series +of corporate voice-mail systems. Kyrie and her friends were +electronic squatters in corporate voice-mail systems, +using them much as if they were pirate bulletin boards, +then moving on when their vocal chatter clogged the system +and the owners necessarily wised up. Kyrie's camp followers +were a loose tribe of some hundred and fifty phone-phreaks, +who followed her trail of piracy from machine to machine, +ardently begging for her services and expertise. + +Kyrie's disciples passed her stolen credit-card numbers, +in exchange for her stolen "long distance information." +Some of Kyrie's clients paid her off in cash, by scamming +credit-card cash advances from Western Union. + +Kyrie travelled incessantly, mostly through airline tickets +and hotel rooms that she scammed through stolen credit cards. +Tiring of this, she found refuge with a fellow female phone +phreak in Chicago. Kyrie's hostess, like a surprising number +of phone phreaks, was blind. She was also physically disabled. +Kyrie allegedly made the best of her new situation by applying for, +and receiving, state welfare funds under a false identity as +a qualified caretaker for the handicapped. + +Sadly, Kyrie's two children by a former marriage had also vanished +underground with her; these pre-teen digital refugees had no legal +American identity, and had never spent a day in school. + +Kyrie was addicted to technical mastery and enthralled by her own +cleverness and the ardent worship of her teenage followers. +This foolishly led her to phone up Gail Thackeray in Arizona, +to boast, brag, strut, and offer to play informant. +Thackeray, however, had already learned far more +than enough about Kyrie, whom she roundly despised +as an adult criminal corrupting minors, a "female Fagin." +Thackeray passed her tapes of Kyrie's boasts to the Secret Service. + +Kyrie was raided and arrested in Chicago in May 1989. +She confessed at great length and pled guilty. + +In August 1990, Cook and his Task Force colleague Colleen Coughlin +sent Kyrie to jail for 27 months, for computer and telecommunications fraud. +This was a markedly severe sentence by the usual wrist-slapping standards +of "hacker" busts. Seven of Kyrie's foremost teenage disciples were also +indicted and convicted. The Kyrie "high-tech street gang," as Cook +described it, had been crushed. Cook and his colleagues had been +the first ever to put someone in prison for voice-mail abuse. +Their pioneering efforts had won them attention and kudos. + +In his article on Kyrie, Cook drove the message home to the readers +of Security Management magazine, a trade journal for corporate +security professionals. The case, Cook said, and Kyrie's stiff sentence, +"reflect a new reality for hackers and computer crime victims in the +'90s. . . . Individuals and corporations who report computer +and telecommunications crimes can now expect that their cooperation +with federal law enforcement will result in meaningful punishment. +Companies and the public at large must report computer-enhanced +crimes if they want prosecutors and the course to protect their rights +to the tangible and intangible property developed and stored on computers." + +Cook had made it his business to construct this "new reality for hackers." +He'd also made it his business to police corporate property rights +to the intangible. + +Had the Electronic Frontier Foundation been a "hacker defense fund" +as that term was generally understood, they presumably would have stood up +for Kyrie. Her 1990 sentence did indeed send a "message" that federal heat +was coming down on "hackers." But Kyrie found no defenders at EFF, +or anywhere else, for that matter. EFF was not a bail-out fund +for electronic crooks. + +The Neidorf case paralleled the Shadowhawk case in certain ways. +The victim once again was allowed to set the value of the "stolen" property. +Once again Kluepfel was both investigator and technical advisor. +Once again no money had changed hands, but the "intent to defraud" was central. + +The prosecution's case showed signs of weakness early on. The Task Force +had originally hoped to prove Neidorf the center of a nationwide +Legion of Doom criminal conspiracy. The Phrack editors threw physical +get-togethers every summer, which attracted hackers from across the country; +generally two dozen or so of the magazine's favorite contributors and readers. +(Such conventions were common in the hacker community; 2600 Magazine, +for instance, held public meetings of hackers in New York, every month.) +LoD heavy-dudes were always a strong presence at these Phrack-sponsored +"Summercons." + +In July 1988, an Arizona hacker named "Dictator" attended Summercon +in Neidorf's home town of St. Louis. Dictator was one of Gail Thackeray's +underground informants; Dictator's underground board in Phoenix was +a sting operation for the Secret Service. Dictator brought an undercover +crew of Secret Service agents to Summercon. The agents bored spyholes +through the wall of Dictator's hotel room in St Louis, and videotaped +the frolicking hackers through a one-way mirror. As it happened, +however, nothing illegal had occurred on videotape, other than the +guzzling of beer by a couple of minors. Summercons were social events, +not sinister cabals. The tapes showed fifteen hours of raucous laughter, +pizza-gobbling, in-jokes and back-slapping. + +Neidorf's lawyer, Sheldon Zenner, saw the Secret Service tapes +before the trial. Zenner was shocked by the complete harmlessness +of this meeting, which Cook had earlier characterized as a sinister +interstate conspiracy to commit fraud. Zenner wanted to show the +Summercon tapes to the jury. It took protracted maneuverings +by the Task Force to keep the tapes from the jury as "irrelevant." + +The E911 Document was also proving a weak reed. It had originally +been valued at $79,449. Unlike Shadowhawk's arcane Artificial Intelligence +booty, the E911 Document was not software--it was written in English. +Computer-knowledgeable people found this value--for a twelve-page +bureaucratic document--frankly incredible. In his "Crime and Puzzlement" +manifesto for EFF, Barlow commented: "We will probably never know how +this figure was reached or by whom, though I like to imagine an appraisal +team consisting of Franz Kafka, Joseph Heller, and Thomas Pynchon." + +As it happened, Barlow was unduly pessimistic. The EFF did, in fact, +eventually discover exactly how this figure was reached, and by whom-- +but only in 1991, long after the Neidorf trial was over. + +Kim Megahee, a Southern Bell security manager, +had arrived at the document's value by simply adding up +the "costs associated with the production" of the E911 Document. +Those "costs" were as follows: + +1. A technical writer had been hired to research and write the E911 Document. + 200 hours of work, at $35 an hour, cost : $7,000. A Project Manager had + overseen the technical writer. 200 hours, at $31 an hour, made: $6,200. + +2. A week of typing had cost $721 dollars. A week of formatting had + cost $721. A week of graphics formatting had cost $742. + +3. Two days of editing cost $367. + +4. A box of order labels cost five dollars. + +5. Preparing a purchase order for the Document, including typing + and the obtaining of an authorizing signature from within the + BellSouth bureaucracy, cost $129. + +6. Printing cost $313. Mailing the Document to fifty people + took fifty hours by a clerk, and cost $858. + +7. Placing the Document in an index took two clerks an hour each, + totalling $43. + +Bureaucratic overhead alone, therefore, was alleged to have cost +a whopping $17,099. According to Mr. Megahee, the typing +of a twelve-page document had taken a full week. Writing it +had taken five weeks, including an overseer who apparently +did nothing else but watch the author for five weeks. +Editing twelve pages had taken two days. Printing and mailing +an electronic document (which was already available on the +Southern Bell Data Network to any telco employee who needed it), +had cost over a thousand dollars. + +But this was just the beginning. There were also the HARDWARE EXPENSES. +Eight hundred fifty dollars for a VT220 computer monitor. +THIRTY-ONE THOUSAND DOLLARS for a sophisticated VAXstation II computer. +Six thousand dollars for a computer printer. TWENTY-TWO THOUSAND DOLLARS +for a copy of "Interleaf" software. Two thousand five hundred dollars +for VMS software. All this to create the twelve-page Document. + +Plus ten percent of the cost of the software and the hardware, for maintenance. +(Actually, the ten percent maintenance costs, though mentioned, had been left +off the final $79,449 total, apparently through a merciful oversight). + +Mr. Megahee's letter had been mailed directly to William Cook himself, +at the office of the Chicago federal attorneys. The United States Government +accepted these telco figures without question. + +As incredulity mounted, the value of the E911 Document was officially +revised downward. This time, Robert Kibler of BellSouth Security +estimated the value of the twelve pages as a mere $24,639.05--based, +purportedly, on "R&D costs." But this specific estimate, +right down to the nickel, did not move the skeptics at all; +in fact it provoked open scorn and a torrent of sarcasm. + +The financial issues concerning theft of proprietary information +have always been peculiar. It could be argued that BellSouth +had not "lost" its E911 Document at all in the first place, +and therefore had not suffered any monetary damage from this "theft." +And Sheldon Zenner did in fact argue this at Neidorf's trial-- +that Prophet's raid had not been "theft," but was better understood +as illicit copying. + +The money, however, was not central to anyone's true purposes in this trial. +It was not Cook's strategy to convince the jury that the E911 Document +was a major act of theft and should be punished for that reason alone. +His strategy was to argue that the E911 Document was DANGEROUS. +It was his intention to establish that the E911 Document was "a road-map" +to the Enhanced 911 System. Neidorf had deliberately and recklessly +distributed a dangerous weapon. Neidorf and the Prophet did not care +(or perhaps even gloated at the sinister idea) that the E911 Document +could be used by hackers to disrupt 911 service, "a life line for every +person certainly in the Southern Bell region of the United States, +and indeed, in many communities throughout the United States," +in Cook's own words. Neidorf had put people's lives in danger. + +In pre-trial maneuverings, Cook had established that the E911 Document +was too hot to appear in the public proceedings of the Neidorf trial. +The JURY ITSELF would not be allowed to ever see this Document, +lest it slip into the official court records, and thus into the hands +of the general public, and, thus, somehow, to malicious hackers +who might lethally abuse it. + +Hiding the E911 Document from the jury may have been a +clever legal maneuver, but it had a severe flaw. There were, +in point of fact, hundreds, perhaps thousands, of people, +already in possession of the E911 Document, just as Phrack +had published it. Its true nature was already obvious +to a wide section of the interested public (all of whom, +by the way, were, at least theoretically, party to +a gigantic wire-fraud conspiracy). Most everyone +in the electronic community who had a modem and any +interest in the Neidorf case already had a copy of the Document. +It had already been available in Phrack for over a year. + +People, even quite normal people without any particular +prurient interest in forbidden knowledge, did not shut their eyes +in terror at the thought of beholding a "dangerous" document +from a telephone company. On the contrary, they tended to trust +their own judgement and simply read the Document for themselves. +And they were not impressed. + +One such person was John Nagle. Nagle was a forty-one-year-old +professional programmer with a masters' degree in computer science +from Stanford. He had worked for Ford Aerospace, where he had invented +a computer-networking technique known as the "Nagle Algorithm," +and for the prominent Californian computer-graphics firm "Autodesk," +where he was a major stockholder. + +Nagle was also a prominent figure on the Well, much respected +for his technical knowledgeability. + +Nagle had followed the civil-liberties debate closely, +for he was an ardent telecommunicator. He was no particular friend +of computer intruders, but he believed electronic publishing +had a great deal to offer society at large, and attempts +to restrain its growth, or to censor free electronic expression, +strongly roused his ire. + +The Neidorf case, and the E911 Document, were both being discussed +in detail on the Internet, in an electronic publication called Telecom Digest. +Nagle, a longtime Internet maven, was a regular reader of Telecom Digest. +Nagle had never seen a copy of Phrack, but the implications of the case +disturbed him. + +While in a Stanford bookstore hunting books on robotics, +Nagle happened across a book called The Intelligent Network. +Thumbing through it at random, Nagle came across an entire chapter +meticulously detailing the workings of E911 police emergency systems. +This extensive text was being sold openly, and yet in Illinois +a young man was in danger of going to prison for publishing +a thin six-page document about 911 service. + +Nagle made an ironic comment to this effect in Telecom Digest. +From there, Nagle was put in touch with Mitch Kapor, +and then with Neidorf's lawyers. + +Sheldon Zenner was delighted to find a computer telecommunications expert +willing to speak up for Neidorf, one who was not a wacky teenage "hacker." +Nagle was fluent, mature, and respectable; he'd once had a federal +security clearance. + +Nagle was asked to fly to Illinois to join the defense team. + +Having joined the defense as an expert witness, Nagle read the entire +E911 Document for himself. He made his own judgement about its potential +for menace. + +The time has now come for you yourself, the reader, to have a look +at the E911 Document. This six-page piece of work was the pretext +for a federal prosecution that could have sent an electronic publisher +to prison for thirty, or even sixty, years. It was the pretext +for the search and seizure of Steve Jackson Games, a legitimate publisher +of printed books. It was also the formal pretext for the search +and seizure of the Mentor's bulletin board, "Phoenix Project," +and for the raid on the home of Erik Bloodaxe. It also had much +to do with the seizure of Richard Andrews' Jolnet node +and the shutdown of Charles Boykin's AT&T node. +The E911 Document was the single most important piece +of evidence in the Hacker Crackdown. There can be no real +and legitimate substitute for the Document itself. + + +==Phrack Inc.== + +Volume Two, Issue 24, File 5 of 13 + +Control Office Administration +Of Enhanced 911 Services For +Special Services and Account Centers + +by the Eavesdropper + +March, 1988 + + +Description of Service +~~~~~~~~~~~~~~~~~~~~~ +The control office for Emergency 911 service is assigned in +accordance with the existing standard guidelines to one of +the following centers: + +o Special Services Center (SSC) +o Major Accounts Center (MAC) +o Serving Test Center (STC) +o Toll Control Center (TCC) + +The SSC/MAC designation is used in this document interchangeably +for any of these four centers. The Special Services Centers (SSCs) +or Major Account Centers (MACs) have been designated as the trouble +reporting contact for all E911 customer (PSAP) reported troubles. +Subscribers who have trouble on an E911 call will continue +to contact local repair service (CRSAB) who will refer the +trouble to the SSC/MAC, when appropriate. + +Due to the critical nature of E911 service, the control +and timely repair of troubles is demanded. As the primary +E911 customer contact, the SSC/MAC is in the unique position +to monitor the status of the trouble and insure its resolution. + +System Overview +~~~~~~~~~~~~~~ +The number 911 is intended as a nationwide universal +telephone number which provides the public with direct +access to a Public Safety Answering Point (PSAP). A PSAP +is also referred to as an Emergency Service Bureau (ESB). +A PSAP is an agency or facility which is authorized by a +municipality to receive and respond to police, fire and/or +ambulance services. One or more attendants are located +at the PSAP facilities to receive and handle calls of an +emergency nature in accordance with the local municipal +requirements. + +An important advantage of E911 emergency service is +improved (reduced) response times for emergency +services. Also close coordination among agencies +providing various emergency services is a valuable +capability provided by E911 service. + +1A ESS is used as the tandem office for the E911 network to +route all 911 calls to the correct (primary) PSAP designated +to serve the calling station. The E911 feature was +developed primarily to provide routing to the correct PSAP +for all 911 calls. Selective routing allows a 911 call +originated from a particular station located in a particular +district, zone, or town, to be routed to the primary PSAP +designated to serve that customer station regardless of +wire center boundaries. Thus, selective routing eliminates +the problem of wire center boundaries not coinciding with +district or other political boundaries. + +The services available with the E911 feature include: + +Forced Disconnect Default Routing +Alternative Routing Night Service +Selective Routing Automatic Number +Identification (ANI) +Selective Transfer Automatic Location +Identification (ALI) + + +Preservice/Installation Guidelines +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +When a contract for an E911 system has been signed, it is +the responsibility of Network Marketing to establish an +implementation/cutover committee which should include +a representative from the SSC/MAC. Duties of the E911 +Implementation Team include coordination of all phases +of the E911 system deployment and the formation of an +on-going E911 maintenance subcommittee. + +Marketing is responsible for providing the following +customer specific information to the SSC/MAC prior to +the start of call through testing: + +o All PSAP's (name, address, local contact) +o All PSAP circuit ID's +o 1004 911 service request including PSAP details on each PSAP + (1004 Section K, L, M) +o Network configuration +o Any vendor information (name, telephone number, equipment) + +The SSC/MAC needs to know if the equipment and sets +at the PSAP are maintained by the BOCs, an independent +company, or an outside vendor, or any combination. +This information is then entered on the PSAP profile sheets +and reviewed quarterly for changes, additions and deletions. + +Marketing will secure the Major Account Number (MAN) +and provide this number to Corporate Communications +so that the initial issue of the service orders carry +the MAN and can be tracked by the SSC/MAC via CORDNET. +PSAP circuits are official services by definition. + +All service orders required for the installation of the E911 +system should include the MAN assigned to the city/county +which has purchased the system. + +In accordance with the basic SSC/MAC strategy for provisioning, +the SSC/MAC will be Overall Control Office (OCO) for all Node +to PSAP circuits (official services) and any other services +for this customer. Training must be scheduled for all SSC/MAC +involved personnel during the pre-service stage of the project. + +The E911 Implementation Team will form the on-going +maintenance subcommittee prior to the initial +implementation of the E911 system. This sub-committee +will establish post implementation quality assurance +procedures to ensure that the E911 system continues to +provide quality service to the customer. +Customer/Company training, trouble reporting interfaces +for the customer, telephone company and any involved +independent telephone companies needs to be addressed +and implemented prior to E911 cutover. These functions +can be best addressed by the formation of a sub- +committee of the E911 Implementation Team to set up +guidelines for and to secure service commitments of +interfacing organizations. A SSC/MAC supervisor should +chair this subcommittee and include the following +organizations: + +1) Switching Control Center + - E911 translations + - Trunking + - End office and Tandem office hardware/software +2) Recent Change Memory Administration Center + - Daily RC update activity for TN/ESN translations + - Processes validity errors and rejects +3) Line and Number Administration + - Verification of TN/ESN translations +4) Special Service Center/Major Account Center + - Single point of contact for all PSAP and Node to host troubles + - Logs, tracks & statusing of all trouble reports + - Trouble referral, follow up, and escalation + - Customer notification of status and restoration + - Analyzation of "chronic" troubles + - Testing, installation and maintenance of E911 circuits +5) Installation and Maintenance (SSIM/I&M) + - Repair and maintenance of PSAP equipment and Telco owned sets +6) Minicomputer Maintenance Operations Center + - E911 circuit maintenance (where applicable) +7) Area Maintenance Engineer + - Technical assistance on voice (CO-PSAP) network related E911 troubles + + +Maintenance Guidelines +~~~~~~~~~~~~~~~~~~~~~ +The CCNC will test the Node circuit from the 202T at the +Host site to the 202T at the Node site. Since Host to Node +(CCNC to MMOC) circuits are official company services, +the CCNC will refer all Node circuit troubles to the +SSC/MAC. The SSC/MAC is responsible for the testing +and follow up to restoration of these circuit troubles. + +Although Node to PSAP circuit are official services, the +MMOC will refer PSAP circuit troubles to the appropriate +SSC/MAC. The SSC/MAC is responsible for testing and +follow up to restoration of PSAP circuit troubles. + +The SSC/MAC will also receive reports from +CRSAB/IMC(s) on subscriber 911 troubles when they are +not line troubles. The SSC/MAC is responsible for testing +and restoration of these troubles. + +Maintenance responsibilities are as follows: + +SCC@ Voice Network (ANI to PSAP) +@SCC responsible for tandem switch + +SSIM/I&M PSAP Equipment (Modems, CIU's, sets) +Vendor PSAP Equipment (when CPE) +SSC/MAC PSAP to Node circuits, and tandem to + PSAP voice circuits (EMNT) +MMOC Node site (Modems, cables, etc) + +Note: All above work groups are required to resolve troubles +by interfacing with appropriate work groups for resolution. + +The Switching Control Center (SCC) is responsible for +E911/1AESS translations in tandem central offices. +These translations route E911 calls, selective transfer, +default routing, speed calling, etc., for each PSAP. +The SCC is also responsible for troubleshooting on +the voice network (call originating to end office tandem equipment). + +For example, ANI failures in the originating offices would +be a responsibility of the SCC. + +Recent Change Memory Administration Center (RCMAC) performs +the daily tandem translation updates (recent change) +for routing of individual telephone numbers. + +Recent changes are generated from service order activity +(new service, address changes, etc.) and compiled into +a daily file by the E911 Center (ALI/DMS E911 Computer). + +SSIM/I&M is responsible for the installation and repair of +PSAP equipment. PSAP equipment includes ANI Controller, +ALI Controller, data sets, cables, sets, and other peripheral +equipment that is not vendor owned. SSIM/I&M is responsible +for establishing maintenance test kits, complete with spare parts +for PSAP maintenance. This includes test gear, data sets, +and ANI/ALI Controller parts. + +Special Services Center (SSC) or Major Account Center +(MAC) serves as the trouble reporting contact for all +(PSAP) troubles reported by customer. The SSC/MAC +refers troubles to proper organizations for handling and +tracks status of troubles, escalating when necessary. +The SSC/MAC will close out troubles with customer. +The SSC/MAC will analyze all troubles and tracks "chronic" +PSAP troubles. + +Corporate Communications Network Center (CCNC) will +test and refer troubles on all node to host circuits. +All E911 circuits are classified as official company property. + +The Minicomputer Maintenance Operations Center +(MMOC) maintains the E911 (ALI/DMS) computer +hardware at the Host site. This MMOC is also responsible +for monitoring the system and reporting certain PSAP +and system problems to the local MMOC's, SCC's or +SSC/MAC's. The MMOC personnel also operate software +programs that maintain the TN data base under the +direction of the E911 Center. The maintenance of the +NODE computer (the interface between the PSAP and the +ALI/DMS computer) is a function of the MMOC at the +NODE site. The MMOC's at the NODE sites may also be +involved in the testing of NODE to Host circuits. +The MMOC will also assist on Host to PSAP and data network +related troubles not resolved through standard trouble +clearing procedures. + +Installation And Maintenance Center (IMC) is responsible +for referral of E911 subscriber troubles that are not subscriber +line problems. + +E911 Center - Performs the role of System Administration +and is responsible for overall operation of the E911 +computer software. The E911 Center does A-Z trouble +analysis and provides statistical information on the +performance of the system. + +This analysis includes processing PSAP inquiries (trouble +reports) and referral of network troubles. The E911 Center +also performs daily processing of tandem recent change +and provides information to the RCMAC for tandem input. +The E911 Center is responsible for daily processing +of the ALI/DMS computer data base and provides error files, +etc. to the Customer Services department for investigation and correction. +The E911 Center participates in all system implementations and on-going +maintenance effort and assists in the development of procedures, +training and education of information to all groups. + +Any group receiving a 911 trouble from the SSC/MAC should +close out the trouble with the SSC/MAC or provide a status +if the trouble has been referred to another group. +This will allow the SSC/MAC to provide a status back +to the customer or escalate as appropriate. + +Any group receiving a trouble from the Host site (MMOC +or CCNC) should close the trouble back to that group. + +The MMOC should notify the appropriate SSC/MAC +when the Host, Node, or all Node circuits are down so that +the SSC/MAC can reply to customer reports that may be +called in by the PSAPs. This will eliminate duplicate +reporting of troubles. On complete outages the MMOC +will follow escalation procedures for a Node after two (2) +hours and for a PSAP after four (4) hours. Additionally the +MMOC will notify the appropriate SSC/MAC when the +Host, Node, or all Node circuits are down. + +The PSAP will call the SSC/MAC to report E911 troubles. +The person reporting the E911 trouble may not have a +circuit I.D. and will therefore report the PSAP name and +address. Many PSAP troubles are not circuit specific. In +those instances where the caller cannot provide a circuit +I.D., the SSC/MAC will be required to determine the +circuit I.D. using the PSAP profile. Under no circumstances +will the SSC/MAC Center refuse to take the trouble. +The E911 trouble should be handled as quickly as possible, +with the SSC/MAC providing as much assistance as +possible while taking the trouble report from the caller. + +The SSC/MAC will screen/test the trouble to determine the +appropriate handoff organization based on the following criteria: + +PSAP equipment problem: SSIM/I&M +Circuit problem: SSC/MAC +Voice network problem: SCC (report trunk group number) +Problem affecting multiple PSAPs (No ALI report from +all PSAPs): Contact the MMOC to check for NODE or +Host computer problems before further testing. + +The SSC/MAC will track the status of reported troubles +and escalate as appropriate. The SSC/MAC will close out +customer/company reports with the initiating contact. +Groups with specific maintenance responsibilities, +defined above, will investigate "chronic" troubles upon +request from the SSC/MAC and the ongoing maintenance subcommittee. + +All "out of service" E911 troubles are priority one type reports. +One link down to a PSAP is considered a priority one trouble +and should be handled as if the PSAP was isolated. + +The PSAP will report troubles with the ANI controller, ALI +controller or set equipment to the SSC/MAC. + +NO ANI: Where the PSAP reports NO ANI (digital +display screen is blank) ask if this condition exists on all +screens and on all calls. It is important to differentiate +between blank screens and screens displaying 911-00XX, +or all zeroes. + +When the PSAP reports all screens on all calls, ask if there +is any voice contact with callers. If there is no voice +contact the trouble should be referred to the SCC +immediately since 911 calls are not getting through which +may require alternate routing of calls to another PSAP. + +When the PSAP reports this condition on all screens +but not all calls and has voice contact with callers, +the report should be referred to SSIM/I&M for dispatch. +The SSC/MAC should verify with the SCC that ANI +is pulsing before dispatching SSIM. + +When the PSAP reports this condition on one screen for +all calls (others work fine) the trouble should be referred +to SSIM/I&M for dispatch, because the trouble is isolated to +one piece of equipment at the customer premise. + +An ANI failure (i.e. all zeroes) indicates that the ANI has +not been received by the PSAP from the tandem office or +was lost by the PSAP ANI controller. The PSAP may +receive "02" alarms which can be caused by the ANI +controller logging more than three all zero failures on the +same trunk. The PSAP has been instructed to report this +condition to the SSC/MAC since it could indicate an +equipment trouble at the PSAP which might be affecting +all subscribers calling into the PSAP. When all zeroes are +being received on all calls or "02" alarms continue, a tester +should analyze the condition to determine the appropriate +action to be taken. The tester must perform cooperative +testing with the SCC when there appears to be a problem +on the Tandem-PSAP trunks before requesting dispatch. + +When an occasional all zero condition is reported, +the SSC/MAC should dispatch SSIM/I&M to routine +equipment on a "chronic" troublesweep. + +The PSAPs are instructed to report incidental ANI failures +to the BOC on a PSAP inquiry trouble ticket (paper) that +is sent to the Customer Services E911 group and forwarded +to E911 center when required. This usually involves only a +particular telephone number and is not a condition that +would require a report to the SSC/MAC. Multiple ANI +failures which our from the same end office (XX denotes +end office), indicate a hard trouble condition may exist +in the end office or end office tandem trunks. The PSAP will +report this type of condition to the SSC/MAC and the +SSC/MAC should refer the report to the SCC responsible +for the tandem office. NOTE: XX is the ESCO (Emergency +Service Number) associated with the incoming 911 trunks +into the tandem. It is important that the C/MAC tell the +SCC what is displayed at the PSAP (i.e. 911-0011) which +indicates to the SCC which end office is in trouble. + +Note: It is essential that the PSAP fill out inquiry form +on every ANI failure. + +The PSAP will report a trouble any time an address is not +received on an address display (screen blank) E911 call. +(If a record is not in the 911 data base or an ANI failure +is encountered, the screen will provide a display noticing +such condition). The SSC/MAC should verify with the PSAP +whether the NO ALI condition is on one screen or all screens. + +When the condition is on one screen (other screens +receive ALI information) the SSC/MAC will request +SSIM/I&M to dispatch. + +If no screens are receiving ALI information, there is usually +a circuit trouble between the PSAP and the Host computer. +The SSC/MAC should test the trouble and refer for restoral. + +Note: If the SSC/MAC receives calls from multiple +PSAP's, all of which are receiving NO ALI, there is a +problem with the Node or Node to Host circuits or the +Host computer itself. Before referring the trouble the +SSC/MAC should call the MMOC to inquire if the Node +or Host is in trouble. + +Alarm conditions on the ANI controller digital display at +the PSAP are to be reported by the PSAP's. These alarms +can indicate various trouble conditions so the SSC/MAC +should ask the PSAP if any portion of the E911 system +is not functioning properly. + +The SSC/MAC should verify with the PSAP attendant that +the equipment's primary function is answering E911 calls. +If it is, the SSC/MAC should request a dispatch SSIM/I&M. +If the equipment is not primarily used for E911, +then the SSC/MAC should advise PSAP to contact their CPE vendor. + +Note: These troubles can be quite confusing when the +PSAP has vendor equipment mixed in with equipment +that the BOC maintains. The Marketing representative +should provide the SSC/MAC information concerning any +unusual or exception items where the PSAP should +contact their vendor. This information should be included +in the PSAP profile sheets. + +ANI or ALI controller down: When the host computer sees +the PSAP equipment down and it does not come back up, +the MMOC will report the trouble to the SSC/MAC; +the equipment is down at the PSAP, a dispatch will be required. + +PSAP link (circuit) down: The MMOC will provide the +SSC/MAC with the circuit ID that the Host computer +indicates in trouble. Although each PSAP has two circuits, +when either circuit is down the condition must be treated +as an emergency since failure of the second circuit will +cause the PSAP to be isolated. + +Any problems that the MMOC identifies from the Node +location to the Host computer will be handled directly +with the appropriate MMOC(s)/CCNC. + +Note: The customer will call only when a problem is +apparent to the PSAP. When only one circuit is down to +the PSAP, the customer may not be aware there is a +trouble, even though there is one link down, +notification should appear on the PSAP screen. +Troubles called into the SSC/MAC from the MMOC +or other company employee should not be closed out +by calling the PSAP since it may result in the +customer responding that they do not have a trouble. +These reports can only be closed out by receiving +information that the trouble was fixed and by checking +with the company employee that reported the trouble. +The MMOC personnel will be able to verify that the +trouble has cleared by reviewing a printout from the host. + +When the CRSAB receives a subscriber complaint +(i.e., cannot dial 911) the RSA should obtain as much +information as possible while the customer is on the line. + +For example, what happened when the subscriber dialed 911? +The report is automatically directed to the IMC for subscriber line testing. +When no line trouble is found, the IMC will refer the trouble condition +to the SSC/MAC. The SSC/MAC will contact Customer Services E911 Group +and verify that the subscriber should be able to call 911 and obtain the ESN. +The SSC/MAC will verify the ESN via 2SCCS. When both verifications match, +the SSC/MAC will refer the report to the SCC responsible for the 911 tandem +office for investigation and resolution. The MAC is responsible for tracking +the trouble and informing the IMC when it is resolved. + + +For more information, please refer to E911 Glossary of Terms. +End of Phrack File +_____________________________________ + + +The reader is forgiven if he or she was entirely unable to read +this document. John Perry Barlow had a great deal of fun at its expense, +in "Crime and Puzzlement:" "Bureaucrat-ese of surpassing opacity. . . . +To read the whole thing straight through without entering coma requires +either a machine or a human who has too much practice thinking like one. +Anyone who can understand it fully and fluidly had altered his consciousness +beyond the ability to ever again read Blake, Whitman, or Tolstoy. . . . +the document contains little of interest to anyone who is not a student +of advanced organizational sclerosis." + +With the Document itself to hand, however, exactly as it was published +(in its six-page edited form) in Phrack, the reader may be able to verify +a few statements of fact about its nature. First, there is no software, +no computer code, in the Document. It is not computer-programming language +like FORTRAN or C++, it is English; all the sentences have nouns and verbs +and punctuation. It does not explain how to break into the E911 system. +It does not suggest ways to destroy or damage the E911 system. + +There are no access codes in the Document. There are no computer passwords. +It does not explain how to steal long distance service. It does not explain +how to break in to telco switching stations. There is nothing in it about +using a personal computer or a modem for any purpose at all, good or bad. + +Close study will reveal that this document is not about machinery. +The E911 Document is about ADMINISTRATION. It describes how one creates +and administers certain units of telco bureaucracy: +Special Service Centers and Major Account Centers (SSC/MAC). +It describes how these centers should distribute responsibility +for the E911 service, to other units of telco bureaucracy, +in a chain of command, a formal hierarchy. It describes +who answers customer complaints, who screens calls, +who reports equipment failures, who answers those reports, +who handles maintenance, who chairs subcommittees, +who gives orders, who follows orders, WHO tells WHOM what to do. +The Document is not a "roadmap" to computers. +The Document is a roadmap to PEOPLE. + +As an aid to breaking into computer systems, the Document is USELESS. +As an aid to harassing and deceiving telco people, however, the Document +might prove handy (especially with its Glossary, which I have not included). +An intense and protracted study of this Document and its Glossary, +combined with many other such documents, might teach one to speak like +a telco employee. And telco people live by SPEECH--they live by phone +communication. If you can mimic their language over the phone, +you can "social-engineer" them. If you can con telco people, you can +wreak havoc among them. You can force them to no longer trust one another; +you can break the telephonic ties that bind their community; you can make +them paranoid. And people will fight harder to defend their community +than they will fight to defend their individual selves. + +This was the genuine, gut-level threat posed by Phrack magazine. +The real struggle was over the control of telco language, +the control of telco knowledge. It was a struggle to defend the social +"membrane of differentiation" that forms the walls of the telco +community's ivory tower --the special jargon that allows telco +professionals to recognize one another, and to exclude charlatans, +thieves, and upstarts. And the prosecution brought out this fact. +They repeatedly made reference to the threat posed to telco professionals +by hackers using "social engineering." + +However, Craig Neidorf was not on trial for learning to speak like +a professional telecommunications expert. Craig Neidorf was on trial +for access device fraud and transportation of stolen property. +He was on trial for stealing a document that was purportedly +highly sensitive and purportedly worth tens of thousands of dollars. + +# + +John Nagle read the E911 Document. He drew his own conclusions. +And he presented Zenner and his defense team with an overflowing box +of similar material, drawn mostly from Stanford University's +engineering libraries. During the trial, the defense team--Zenner, +half-a-dozen other attorneys, Nagle, Neidorf, and computer-security +expert Dorothy Denning, all pored over the E911 Document line-by-line. + +On the afternoon of July 25, 1990, Zenner began to cross-examine +a woman named Billie Williams, a service manager for Southern Bell +in Atlanta. Ms. Williams had been responsible for the E911 Document. +(She was not its author--its original "author" was a Southern Bell +staff manager named Richard Helms. However, Mr. Helms should not bear +the entire blame; many telco staff people and maintenance personnel +had amended the Document. It had not been so much "written" by a +single author, as built by committee out of concrete-blocks of jargon.) + +Ms. Williams had been called as a witness for the prosecution, +and had gamely tried to explain the basic technical structure +of the E911 system, aided by charts. + +Now it was Zenner's turn. He first established that the +"proprietary stamp" that BellSouth had used on the E911 Document +was stamped on EVERY SINGLE DOCUMENT that BellSouth wrote-- +THOUSANDS of documents. "We do not publish anything other +than for our own company," Ms. Williams explained. +"Any company document of this nature is considered proprietary." +Nobody was in charge of singling out special high-security publications +for special high-security protection. They were ALL special, +no matter how trivial, no matter what their subject matter-- +the stamp was put on as soon as any document was written, +and the stamp was never removed. + +Zenner now asked whether the charts she had been using to explain +the mechanics of E911 system were "proprietary," too. +Were they PUBLIC INFORMATION, these charts, all about PSAPs, +ALIs, nodes, local end switches? Could he take the charts out +in the street and show them to anybody, "without violating +some proprietary notion that BellSouth has?" + +Ms Williams showed some confusion, but finally areed that the charts were, +in fact, public. + +"But isn't this what you said was basically what appeared in Phrack?" + +Ms. Williams denied this. + +Zenner now pointed out that the E911 Document as published in Phrack +was only half the size of the original E911 Document (as Prophet +had purloined it). Half of it had been deleted--edited by Neidorf. + +Ms. Williams countered that "Most of the information that is +in the text file is redundant." + +Zenner continued to probe. Exactly what bits of knowledge in the Document +were, in fact, unknown to the public? Locations of E911 computers? +Phone numbers for telco personnel? Ongoing maintenance subcommittees? +Hadn't Neidorf removed much of this? + +Then he pounced. "Are you familiar with Bellcore Technical Reference +Document TR-TSY-000350?" It was, Zenner explained, officially titled +"E911 Public Safety Answering Point Interface Between 1-1AESS Switch +and Customer Premises Equipment." It contained highly detailed +and specific technical information about the E911 System. +It was published by Bellcore and publicly available for about $20. + +He showed the witness a Bellcore catalog which listed thousands +of documents from Bellcore and from all the Baby Bells, BellSouth included. +The catalog, Zenner pointed out, was free. Anyone with a credit card +could call the Bellcore toll-free 800 number and simply order any +of these documents, which would be shipped to any customer without question. +Including, for instance, "BellSouth E911 Service Interfaces to +Customer Premises Equipment at a Public Safety Answering Point." + +Zenner gave the witness a copy of "BellSouth E911 Service Interfaces," +which cost, as he pointed out, $13, straight from the catalog. +"Look at it carefully," he urged Ms. Williams, "and tell me +if it doesn't contain about twice as much detailed information +about the E911 system of BellSouth than appeared anywhere in Phrack." + +"You want me to. . . ." Ms. Williams trailed off. "I don't understand." + +"Take a careful look," Zenner persisted. "Take a look at that document, +and tell me when you're done looking at it if, indeed, it doesn't contain +much more detailed information about the E911 system than appeared in Phrack." + +"Phrack wasn't taken from this," Ms. Williams said. + +"Excuse me?" said Zenner. + +"Phrack wasn't taken from this." + +"I can't hear you," Zenner said. + +"Phrack was not taken from this document. I don't understand +your question to me." + +"I guess you don't," Zenner said. + +At this point, the prosecution's case had been gutshot. +Ms. Williams was distressed. Her confusion was quite genuine. +Phrack had not been taken from any publicly available Bellcore document. +Phrack's E911 Document had been stolen from her own company's computers, +from her own company's text files, that her own colleagues had written, +and revised, with much labor. + +But the "value" of the Document had been blown to smithereens. +It wasn't worth eighty grand. According to Bellcore it was worth +thirteen bucks. And the looming menace that it supposedly posed +had been reduced in instants to a scarecrow. Bellcore itself +was selling material far more detailed and "dangerous," +to anybody with a credit card and a phone. + +Actually, Bellcore was not giving this information to just anybody. +They gave it to ANYBODY WHO ASKED, but not many did ask. +Not many people knew that Bellcore had a free catalog and an 800 number. +John Nagle knew, but certainly the average teenage phreak didn't know. +"Tuc," a friend of Neidorf's and sometime Phrack contributor, knew, +and Tuc had been very helpful to the defense, behind the scenes. +But the Legion of Doom didn't know--otherwise, they would never +have wasted so much time raiding dumpsters. Cook didn't know. +Foley didn't know. Kluepfel didn't know. The right hand +of Bellcore knew not what the left hand was doing. The right +hand was battering hackers without mercy, while the left hand +was distributing Bellcore's intellectual property to anybody +who was interested in telephone technical trivia--apparently, +a pathetic few. + +The digital underground was so amateurish and poorly organized +that they had never discovered this heap of unguarded riches. +The ivory tower of the telcos was so wrapped-up in the fog +of its own technical obscurity that it had left all the +windows open and flung open the doors. No one had even noticed. + +Zenner sank another nail in the coffin. He produced a printed issue +of Telephone Engineer & Management, a prominent industry journal +that comes out twice a month and costs $27 a year. This particular issue +of TE&M, called "Update on 911," featured a galaxy of technical details +on 911 service and a glossary far more extensive than Phrack's. + +The trial rumbled on, somehow, through its own momentum. +Tim Foley testified about his interrogations of Neidorf. +Neidorf's written admission that he had known the E911 Document +was pilfered was officially read into the court record. + +An interesting side issue came up: "Terminus" had once passed Neidorf +a piece of UNIX AT&T software, a log-in sequence, that had been cunningly +altered so that it could trap passwords. The UNIX software itself was +illegally copied AT&T property, and the alterations "Terminus" had made to it, +had transformed it into a device for facilitating computer break-ins. Terminus +himself would eventually plead guilty to theft of this piece of software, +and the Chicago group would send Terminus to prison for it. But it was +of dubious relevance in the Neidorf case. Neidorf hadn't written the program. +He wasn't accused of ever having used it. And Neidorf wasn't being charged +with software theft or owning a password trapper. + +On the next day, Zenner took the offensive. The civil libertarians +now had their own arcane, untried legal weaponry to launch into action-- +the Electronic Communications Privacy Act of 1986, 18 US Code, +Section 2701 et seq. Section 2701 makes it a crime to intentionally +access without authorization a facility in which an electronic communication +service is provided--it is, at heart, an anti-bugging and anti-tapping law, +intended to carry the traditional protections of telephones into other +electronic channels of communication. While providing penalties for amateur +snoops, however, Section 2703 of the ECPA also lays some formal difficulties +on the bugging and tapping activities of police. + +The Secret Service, in the person of Tim Foley, had served Richard Andrews +with a federal grand jury subpoena, in their pursuit of Prophet, +the E911 Document, and the Terminus software ring. But according to +the Electronic Communications Privacy Act, a "provider of remote +computing service" was legally entitled to "prior notice" from +the government if a subpoena was used. Richard Andrews and his +basement UNIX node, Jolnet, had not received any "prior notice." +Tim Foley had purportedly violated the ECPA and committed +an electronic crime! Zenner now sought the judge's permission +to cross-examine Foley on the topic of Foley's own electronic misdeeds. + +Cook argued that Richard Andrews' Jolnet was a privately owned +bulletin board, and not within the purview of ECPA. Judge Bua +granted the motion of the government to prevent cross-examination +on that point, and Zenner's offensive fizzled. This, however, +was the first direct assault on the legality of the actions +of the Computer Fraud and Abuse Task Force itself-- +the first suggestion that they themselves had broken the law, +and might, perhaps, be called to account. + +Zenner, in any case, did not really need the ECPA. +Instead, he grilled Foley on the glaring contradictions in +the supposed value of the E911 Document. He also brought up +the embarrassing fact that the supposedly red-hot E911 Document +had been sitting around for months, in Jolnet, with Kluepfel's knowledge, +while Kluepfel had done nothing about it. + +In the afternoon, the Prophet was brought in to testify +for the prosecution. (The Prophet, it will be recalled, +had also been indicted in the case as partner in a fraud +scheme with Neidorf.) In Atlanta, the Prophet had already +pled guilty to one charge of conspiracy, one charge of wire fraud +and one charge of interstate transportation of stolen property. +The wire fraud charge, and the stolen property charge, +were both directly based on the E911 Document. + +The twenty-year-old Prophet proved a sorry customer, +answering questions politely but in a barely audible mumble, +his voice trailing off at the ends of sentences. +He was constantly urged to speak up. + +Cook, examining Prophet, forced him to admit that +he had once had a "drug problem," abusing amphetamines, +marijuana, cocaine, and LSD. This may have established +to the jury that "hackers" are, or can be, seedy lowlife characters, +but it may have damaged Prophet's credibility somewhat. +Zenner later suggested that drugs might have damaged Prophet's memory. +The interesting fact also surfaced that Prophet had never +physically met Craig Neidorf. He didn't even know +Neidorf's last name--at least, not until the trial. + +Prophet confirmed the basic facts of his hacker career. +He was a member of the Legion of Doom. He had abused codes, +he had broken into switching stations and re-routed calls, +he had hung out on pirate bulletin boards. He had raided +the BellSouth AIMSX computer, copied the E911 Document, +stored it on Jolnet, mailed it to Neidorf. He and Neidorf +had edited it, and Neidorf had known where it came from. + +Zenner, however, had Prophet confirm that Neidorf was not a member +of the Legion of Doom, and had not urged Prophet to break into +BellSouth computers. Neidorf had never urged Prophet to defraud anyone, +or to steal anything. Prophet also admitted that he had never known Neidorf +to break in to any computer. Prophet said that no one in the Legion of Doom +considered Craig Neidorf a "hacker" at all. Neidorf was not a UNIX maven, +and simply lacked the necessary skill and ability to break into computers. +Neidorf just published a magazine. + +On Friday, July 27, 1990, the case against Neidorf collapsed. +Cook moved to dismiss the indictment, citing "information currently +available to us that was not available to us at the inception of the trial." +Judge Bua praised the prosecution for this action, which he described as +"very responsible," then dismissed a juror and declared a mistrial. + +Neidorf was a free man. His defense, however, had cost himself +and his family dearly. Months of his life had been consumed in anguish; +he had seen his closest friends shun him as a federal criminal. +He owed his lawyers over a hundred thousand dollars, despite +a generous payment to the defense by Mitch Kapor. + +Neidorf was not found innocent. The trial was simply dropped. +Nevertheless, on September 9, 1991, Judge Bua granted Neidorf's +motion for the "expungement and sealing" of his indictment record. +The United States Secret Service was ordered to delete and destroy +all fingerprints, photographs, and other records of arrest +or processing relating to Neidorf's indictment, including +their paper documents and their computer records. + +Neidorf went back to school, blazingly determined to become a lawyer. +Having seen the justice system at work, Neidorf lost much of his enthusiasm +for merely technical power. At this writing, Craig Neidorf is working +in Washington as a salaried researcher for the American Civil Liberties Union. + +# + +The outcome of the Neidorf trial changed the EFF +from voices-in-the-wilderness to the media darlings +of the new frontier. + +Legally speaking, the Neidorf case was not a sweeping triumph +for anyone concerned. No constitutional principles had been established. +The issues of "freedom of the press" for electronic publishers remained +in legal limbo. There were public misconceptions about the case. +Many people thought Neidorf had been found innocent and relieved +of all his legal debts by Kapor. The truth was that the government +had simply dropped the case, and Neidorf's family had gone deeply +into hock to support him. + +But the Neidorf case did provide a single, devastating, public sound-bite: +THE FEDS SAID IT WAS WORTH EIGHTY GRAND, AND IT WAS ONLY WORTH THIRTEEN BUCKS. + +This is the Neidorf case's single most memorable element. No serious report +of the case missed this particular element. Even cops could not read this +without a wince and a shake of the head. It left the public credibility +of the crackdown agents in tatters. + +The crackdown, in fact, continued, however. Those two charges +against Prophet, which had been based on the E911 Document, +were quietly forgotten at his sentencing--even though Prophet +had already pled guilty to them. Georgia federal prosecutors +strongly argued for jail time for the Atlanta Three, insisting on +"the need to send a message to the community," "the message that +hackers around the country need to hear." + +There was a great deal in their sentencing memorandum +about the awful things that various other hackers had done +(though the Atlanta Three themselves had not, in fact, +actually committed these crimes). There was also much +speculation about the awful things that the Atlanta Three +MIGHT have done and WERE CAPABLE of doing (even though +they had not, in fact, actually done them). +The prosecution's argument carried the day. +The Atlanta Three were sent to prison: +Urvile and Leftist both got 14 months each, +while Prophet (a second offender) got 21 months. + +The Atlanta Three were also assessed staggering fines as "restitution": +$233,000 each. BellSouth claimed that the defendants had "stolen" +"approximately $233,880 worth" of "proprietary computer access information"-- +specifically, $233,880 worth of computer passwords and connect addresses. +BellSouth's astonishing claim of the extreme value of its own computer +passwords and addresses was accepted at face value by the Georgia court. +Furthermore (as if to emphasize its theoretical nature) this enormous sum +was not divvied up among the Atlanta Three, but each of them had to pay +all of it. + +A striking aspect of the sentence was that the Atlanta Three were +specifically forbidden to use computers, except for work or under supervision. +Depriving hackers of home computers and modems makes some sense if one +considers hackers as "computer addicts," but EFF, filing an amicus brief +in the case, protested that this punishment was unconstitutional-- +it deprived the Atlanta Three of their rights of free association +and free expression through electronic media. + +Terminus, the "ultimate hacker," was finally sent to prison for a year +through the dogged efforts of the Chicago Task Force. His crime, +to which he pled guilty, was the transfer of the UNIX password trapper, +which was officially valued by AT&T at $77,000, a figure which aroused +intense skepticism among those familiar with UNIX "login.c" programs. + +The jailing of Terminus and the Atlanta Legionnaires of Doom, however, +did not cause the EFF any sense of embarrassment or defeat. +On the contrary, the civil libertarians were rapidly gathering strength. + +An early and potent supporter was Senator Patrick Leahy, +Democrat from Vermont, who had been a Senate sponsor +of the Electronic Communications Privacy Act. Even before +the Neidorf trial, Leahy had spoken out in defense of hacker-power +and freedom of the keyboard: "We cannot unduly inhibit the inquisitive +13-year-old who, if left to experiment today, may tomorrow develop +the telecommunications or computer technology to lead the United States +into the 21st century. He represents our future and our best hope +to remain a technologically competitive nation." + +It was a handsome statement, rendered perhaps rather more effective +by the fact that the crackdown raiders DID NOT HAVE any Senators +speaking out for THEM. On the contrary, their highly secretive +actions and tactics, all "sealed search warrants" here and +"confidential ongoing investigations" there, might have won +them a burst of glamorous publicity at first, but were crippling +them in the on-going propaganda war. Gail Thackeray was reduced +to unsupported bluster: "Some of these people who are loudest +on the bandwagon may just slink into the background," +she predicted in Newsweek--when all the facts came out, +and the cops were vindicated. + +But all the facts did not come out. Those facts that did, +were not very flattering. And the cops were not vindicated. +And Gail Thackeray lost her job. By the end of 1991, +William Cook had also left public employment. + +1990 had belonged to the crackdown, but by '91 its agents +were in severe disarray, and the libertarians were on a roll. +People were flocking to the cause. + +A particularly interesting ally had been Mike Godwin of Austin, Texas. +Godwin was an individual almost as difficult to describe as Barlow; +he had been editor of the student newspaper of the University of Texas, +and a computer salesman, and a programmer, and in 1990 was back +in law school, looking for a law degree. + +Godwin was also a bulletin board maven. He was very well-known +in the Austin board community under his handle "Johnny Mnemonic," +which he adopted from a cyberpunk science fiction story by William Gibson. +Godwin was an ardent cyberpunk science fiction fan. As a fellow Austinite +of similar age and similar interests, I myself had known Godwin socially +for many years. When William Gibson and myself had been writing our +collaborative SF novel, The Difference Engine, Godwin had been our +technical advisor in our effort to link our Apple word-processors +from Austin to Vancouver. Gibson and I were so pleased by his generous +expert help that we named a character in the novel "Michael Godwin" +in his honor. + +The handle "Mnemonic" suited Godwin very well. His erudition +and his mastery of trivia were impressive to the point of stupor; +his ardent curiosity seemed insatiable, and his desire to debate +and argue seemed the central drive of his life. Godwin had even +started his own Austin debating society, wryly known as the +"Dull Men's Club." In person, Godwin could be overwhelming; +a flypaper-brained polymath who could not seem to let any idea go. +On bulletin boards, however, Godwin's closely reasoned, +highly grammatical, erudite posts suited the medium well, +and he became a local board celebrity. + +Mike Godwin was the man most responsible for the public national exposure +of the Steve Jackson case. The Izenberg seizure in Austin had received +no press coverage at all. The March 1 raids on Mentor, Bloodaxe, and +Steve Jackson Games had received a brief front-page splash in the +front page of the Austin American-Statesman, but it was confused +and ill-informed: the warrants were sealed, and the Secret Service +wasn't talking. Steve Jackson seemed doomed to obscurity. +Jackson had not been arrested; he was not charged with any crime; +he was not on trial. He had lost some computers in an ongoing +investigation--so what? Jackson tried hard to attract attention +to the true extent of his plight, but he was drawing a blank; +no one in a position to help him seemed able to get a mental grip +on the issues. + +Godwin, however, was uniquely, almost magically, qualified +to carry Jackson's case to the outside world. Godwin was +a board enthusiast, a science fiction fan, a former journalist, +a computer salesman, a lawyer-to-be, and an Austinite. +Through a coincidence yet more amazing, in his last year +of law school Godwin had specialized in federal prosecutions +and criminal procedure. Acting entirely on his own, Godwin made +up a press packet which summarized the issues and provided useful +contacts for reporters. Godwin's behind-the-scenes effort +(which he carried out mostly to prove a point in a local board debate) +broke the story again in the Austin American-Statesman and then in Newsweek. + +Life was never the same for Mike Godwin after that. As he joined the growing +civil liberties debate on the Internet, it was obvious to all parties involved +that here was one guy who, in the midst of complete murk and confusion, +GENUINELY UNDERSTOOD EVERYTHING HE WAS TALKING ABOUT. The disparate elements +of Godwin's dilettantish existence suddenly fell together as neatly as +the facets of a Rubik's cube. + +When the time came to hire a full-time EFF staff attorney, +Godwin was the obvious choice. He took the Texas bar exam, +left Austin, moved to Cambridge, became a full-time, professional, +computer civil libertarian, and was soon touring the nation on behalf +of EFF, delivering well-received addresses on the issues to crowds +as disparate as academics, industrialists, science fiction fans, +and federal cops. + +Michael Godwin is currently the chief legal counsel of +the Electronic Frontier Foundation in Cambridge, Massachusetts. + +# + +Another early and influential participant in the controversy +was Dorothy Denning. Dr. Denning was unique among investigators +of the computer underground in that she did not enter the debate +with any set of politicized motives. She was a professional +cryptographer and computer security expert whose primary interest +in hackers was SCHOLARLY. She had a B.A. and M.A. in mathematics, +and a Ph.D. in computer science from Purdue. She had worked for SRI +International, the California think-tank that was also the home of +computer-security maven Donn Parker, and had authored an influential text +called Cryptography and Data Security. In 1990, Dr. Denning was working for +Digital Equipment Corporation in their Systems Reseach Center. Her husband, +Peter Denning, was also a computer security expert, working for NASA's +Research Institute for Advanced Computer Science. He had edited the +well-received Computers Under Attack: Intruders, Worms and Viruses. + +Dr. Denning took it upon herself to contact the digital underground, +more or less with an anthropological interest. There she discovered +that these computer-intruding hackers, who had been characterized +as unethical, irresponsible, and a serious danger to society, +did in fact have their own subculture and their own rules. +They were not particularly well-considered rules, but they were, +in fact, rules. Basically, they didn't take money and they +didn't break anything. + +Her dispassionate reports on her researches did a great deal +to influence serious-minded computer professionals--the sort +of people who merely rolled their eyes at the cyberspace +rhapsodies of a John Perry Barlow. + +For young hackers of the digital underground, meeting Dorothy Denning +was a genuinely mind-boggling experience. Here was this neatly coiffed, +conservatively dressed, dainty little personage, who reminded most +hackers of their moms or their aunts. And yet she was an IBM systems +programmer with profound expertise in computer architectures +and high-security information flow, who had personal friends +in the FBI and the National Security Agency. + +Dorothy Denning was a shining example of the American mathematical +intelligentsia, a genuinely brilliant person from the central ranks +of the computer-science elite. And here she was, gently questioning +twenty-year-old hairy-eyed phone-phreaks over the deeper ethical +implications of their behavior. + +Confronted by this genuinely nice lady, most hackers sat up very straight +and did their best to keep the anarchy-file stuff down to a faint whiff +of brimstone. Nevertheless, the hackers WERE in fact prepared to seriously +discuss serious issues with Dorothy Denning. They were willing to speak +the unspeakable and defend the indefensible, to blurt out their convictions +that information cannot be owned, that the databases of governments and large +corporations were a threat to the rights and privacy of individuals. + +Denning's articles made it clear to many that "hacking" +was not simple vandalism by some evil clique of psychotics. +"Hacking" was not an aberrant menace that could be charmed away +by ignoring it, or swept out of existence by jailing a few ringleaders. +Instead, "hacking" was symptomatic of a growing, primal struggle over +knowledge and power in the age of information. + +Denning pointed out that the attitude of hackers were at least partially +shared by forward-looking management theorists in the business community: +people like Peter Drucker and Tom Peters. Peter Drucker, in his book +The New Realities, had stated that "control of information by the government +is no longer possible. Indeed, information is now transnational. +Like money, it has no `fatherland.'" + +And management maven Tom Peters had chided large corporations for uptight, +proprietary attitudes in his bestseller, Thriving on Chaos: +"Information hoarding, especially by politically motivated, +power-seeking staffs, had been commonplace throughout American industry, +service and manufacturing alike. It will be an impossible +millstone aroung the neck of tomorrow's organizations." + +Dorothy Denning had shattered the social membrane of the +digital underground. She attended the Neidorf trial, +where she was prepared to testify for the defense as an expert witness. +She was a behind-the-scenes organizer of two of the most important +national meetings of the computer civil libertarians. Though not +a zealot of any description, she brought disparate elements of the +electronic community into a surprising and fruitful collusion. + +Dorothy Denning is currently the Chair of the Computer Science Department +at Georgetown University in Washington, DC. + +# + +There were many stellar figures in the civil libertarian community. +There's no question, however, that its single most influential figure +was Mitchell D. Kapor. Other people might have formal titles, +or governmental positions, have more experience with crime, +or with the law, or with the arcanities of computer security +or constitutional theory. But by 1991 Kapor had transcended +any such narrow role. Kapor had become "Mitch." + +Mitch had become the central civil-libertarian ad-hocrat. +Mitch had stood up first, he had spoken out loudly, directly, +vigorously and angrily, he had put his own reputation, +and his very considerable personal fortune, on the line. +By mid-'91 Kapor was the best-known advocate of his cause +and was known PERSONALLY by almost every single human being in America +with any direct influence on the question of civil liberties in cyberspace. +Mitch had built bridges, crossed voids, changed paradigms, forged metaphors, +made phone-calls and swapped business cards to such spectacular effect +that it had become impossible for anyone to take any action in the +"hacker question" without wondering what Mitch might think-- +and say--and tell his friends. + +The EFF had simply NETWORKED the situation into an entirely new status quo. +And in fact this had been EFF's deliberate strategy from the beginning. +Both Barlow and Kapor loathed bureaucracies and had deliberately +chosen to work almost entirely through the electronic spiderweb of +"valuable personal contacts." + +After a year of EFF, both Barlow and Kapor had every reason +to look back with satisfaction. EFF had established its own Internet node, +"eff.org," with a well-stocked electronic archive of documents on +electronic civil rights, privacy issues, and academic freedom. +EFF was also publishing EFFector, a quarterly printed journal, +as well as EFFector Online, an electronic newsletter with +over 1,200 subscribers. And EFF was thriving on the Well. + +EFF had a national headquarters in Cambridge and a full-time staff. +It had become a membership organization and was attracting +grass-roots support. It had also attracted the support +of some thirty civil-rights lawyers, ready and eager +to do pro bono work in defense of the Constitution in Cyberspace. + +EFF had lobbied successfully in Washington and in Massachusetts +to change state and federal legislation on computer networking. +Kapor in particular had become a veteran expert witness, +and had joined the Computer Science and Telecommunications Board +of the National Academy of Science and Engineering. + +EFF had sponsored meetings such as "Computers, Freedom and Privacy" +and the CPSR Roundtable. It had carried out a press offensive that, +in the words of EFFector, "has affected the climate of opinion about +computer networking and begun to reverse the slide into +`hacker hysteria' that was beginning to grip the nation." + +It had helped Craig Neidorf avoid prison. + +And, last but certainly not least, the Electronic Frontier Foundation +had filed a federal lawsuit in the name of Steve Jackson, +Steve Jackson Games Inc., and three users of the Illuminati +bulletin board system. The defendants were, and are, +the United States Secret Service, William Cook, Tim Foley, +Barbara Golden and Henry Kleupfel. + +The case, which is in pre-trial procedures in an Austin federal court +as of this writing, is a civil action for damages to redress +alleged violations of the First and Fourth Amendments to the +United States Constitution, as well as the Privacy Protection Act +of 1980 (42 USC 2000aa et seq.), and the Electronic Communications +Privacy Act (18 USC 2510 et seq and 2701 et seq). + +EFF had established that it had credibility. It had also established +that it had teeth. + +In the fall of 1991 I travelled to Massachusetts to speak personally +with Mitch Kapor. It was my final interview for this book. + +# + +The city of Boston has always been one of the major intellectual centers +of the American republic. It is a very old city by American standards, +a place of skyscrapers overshadowing seventeenth-century graveyards, +where the high-tech start-up companies of Route 128 co-exist with the +hand-wrought pre-industrial grace of "Old Ironsides," the USS CONSTITUTION. + +The Battle of Bunker Hill, one of the first and bitterest armed clashes +of the American Revolution, was fought in Boston's environs. Today there is +a monumental spire on Bunker Hill, visible throughout much of the city. +The willingness of the republican revolutionaries to take up arms and fire +on their oppressors has left a cultural legacy that two full centuries +have not effaced. Bunker Hill is still a potent center of American political +symbolism, and the Spirit of '76 is still a potent image for those who seek +to mold public opinion. + +Of course, not everyone who wraps himself in the flag is necessarily +a patriot. When I visited the spire in September 1991, it bore a huge, +badly-erased, spray-can grafitto around its bottom reading +"BRITS OUT--IRA PROVOS." Inside this hallowed edifice was +a glass-cased diorama of thousands of tiny toy soldiers, +rebels and redcoats, fighting and dying over the green hill, +the riverside marshes, the rebel trenchworks. Plaques indicated the +movement of troops, the shiftings of strategy. The Bunker Hill Monument +is occupied at its very center by the toy soldiers of a military +war-game simulation. + +The Boston metroplex is a place of great universities, +prominent among the Massachusetts Institute of Technology, +where the term "computer hacker" was first coined. The Hacker Crackdown +of 1990 might be interpreted as a political struggle among American cities: +traditional strongholds of longhair intellectual liberalism, +such as Boston, San Francisco, and Austin, versus the bare-knuckle +industrial pragmatism of Chicago and Phoenix (with Atlanta and New York +wrapped in internal struggle). + +The headquarters of the Electronic Frontier Foundation is on +155 Second Street in Cambridge, a Bostonian suburb north +of the River Charles. Second Street has weedy sidewalks of dented, +sagging brick and elderly cracked asphalt; large street-signs warn +"NO PARKING DURING DECLARED SNOW EMERGENCY." This is an old area +of modest manufacturing industries; the EFF is catecorner from the +Greene Rubber Company. EFF's building is two stories of red brick; +its large wooden windows feature gracefully arched tops and stone sills. + +The glass window beside the Second Street entrance bears three sheets +of neatly laser-printed paper, taped against the glass. They read: +ON Technology. EFF. KEI. + +"ON Technology" is Kapor's software company, which currently specializes +in "groupware" for the Apple Macintosh computer. "Groupware" is intended +to promote efficient social interaction among office-workers linked +by computers. ON Technology's most successful software products to date +are "Meeting Maker" and "Instant Update." + +"KEI" is Kapor Enterprises Inc., Kapor's personal holding company, +the commercial entity that formally controls his extensive investments +in other hardware and software corporations. + +"EFF" is a political action group--of a special sort. + +Inside, someone's bike has been chained to the handrails +of a modest flight of stairs. A wall of modish glass brick +separates this anteroom from the offices. Beyond the brick, +there's an alarm system mounted on the wall, a sleek, complex little +number that resembles a cross between a thermostat and a CD player. +Piled against the wall are box after box of a recent special issue +of Scientific American, "How to Work, Play, and Thrive in Cyberspace," +with extensive coverage of electronic networking techniques +and political issues, including an article by Kapor himself. +These boxes are addressed to Gerard Van der Leun, EFF's +Director of Communications, who will shortly mail those magazines +to every member of the EFF. + +The joint headquarters of EFF, KEI, and ON Technology, +which Kapor currently rents, is a modestly bustling place. +It's very much the same physical size as Steve Jackson's gaming company. +It's certainly a far cry from the gigantic gray steel-sided railway +shipping barn, on the Monsignor O'Brien Highway, that is owned +by Lotus Development Corporation. + +Lotus is, of course, the software giant that Mitchell Kapor founded +in the late 70s. The software program Kapor co-authored, +"Lotus 1-2-3," is still that company's most profitable product. +"Lotus 1-2-3" also bears a singular distinction in the +digital underground: it's probably the most pirated piece +of application software in world history. + +Kapor greets me cordially in his own office, down a hall. +Kapor, whose name is pronounced KAY-por, is in his early forties, +married and the father of two. He has a round face, high forehead, +straight nose, a slightly tousled mop of black hair peppered with gray. +His large brown eyes are wideset, reflective, one might almost say soulful. +He disdains ties, and commonly wears Hawaiian shirts and tropical prints, +not so much garish as simply cheerful and just that little bit anomalous. + +There is just the whiff of hacker brimstone about Mitch Kapor. +He may not have the hard-riding, hell-for-leather, guitar-strumming +charisma of his Wyoming colleague John Perry Barlow, but there's +something about the guy that still stops one short. He has the air +of the Eastern city dude in the bowler hat, the dreamy, +Longfellow-quoting poker shark who only HAPPENS to know +the exact mathematical odds against drawing to an inside straight. +Even among his computer-community colleagues, who are hardly known +for mental sluggishness, Kapor strikes one forcefully as a very +intelligent man. He speaks rapidly, with vigorous gestures, +his Boston accent sometimes slipping to the sharp nasal tang +of his youth in Long Island. + +Kapor, whose Kapor Family Foundation does much of his philanthropic work, +is a strong supporter of Boston's Computer Museum. Kapor's interest +in the history of his industry has brought him some remarkable curios, +such as the "byte" just outside his office door. This "byte"-- +eight digital bits--has been salvaged from the wreck of an +electronic computer of the pre-transistor age. It's a standing gunmetal +rack about the size of a small toaster-oven: with eight slots +of hand-soldered breadboarding featuring thumb-sized vacuum tubes. +If it fell off a table it could easily break your foot, +but it was state-of-the-art computation in the 1940s. +(It would take exactly 157,184 of these primordial toasters +to hold the first part of this book.) + +There's also a coiling, multicolored, scaly dragon that some +inspired techno-punk artist has cobbled up entirely out of transistors, +capacitors, and brightly plastic-coated wiring. + +Inside the office, Kapor excuses himself briefly to do a little +mouse-whizzing housekeeping on his personal Macintosh IIfx. +If its giant screen were an open window, an agile person +could climb through it without much trouble at all. +There's a coffee-cup at Kapor's elbow, a memento of his +recent trip to Eastern Europe, which has a black-and-white +stencilled photo and the legend CAPITALIST FOOLS TOUR. +It's Kapor, Barlow, and two California venture-capitalist luminaries +of their acquaintance, four windblown, grinning Baby Boomer +dudes in leather jackets, boots, denim, travel bags, +standing on airport tarmac somewhere behind the formerly Iron Curtain. +They look as if they're having the absolute time of their lives. + +Kapor is in a reminiscent mood. We talk a bit about his youth-- +high school days as a "math nerd," Saturdays attending Columbia University's +high-school science honors program, where he had his first experience +programming computers. IBM 1620s, in 1965 and '66. "I was very interested," +says Kapor, "and then I went off to college and got distracted by drugs sex +and rock and roll, like anybody with half a brain would have then!" +After college he was a progressive-rock DJ in Hartford, Connecticut, +for a couple of years. + +I ask him if he ever misses his rock and roll days--if he ever wished +he could go back to radio work. + +He shakes his head flatly. "I stopped thinking about going back +to be a DJ the day after Altamont." + +Kapor moved to Boston in 1974 and got a job programming mainframes in COBOL. +He hated it. He quit and became a teacher of transcendental meditation. +(It was Kapor's long flirtation with Eastern mysticism that gave the +world "Lotus.") + +In 1976 Kapor went to Switzerland, where the Transcendental Meditation +movement had rented a gigantic Victorian hotel in St-Moritz. It was +an all-male group--a hundred and twenty of them--determined upon +Enlightenment or Bust. Kapor had given the transcendant his best shot. +He was becoming disenchanted by "the nuttiness in the organization." +"They were teaching people to levitate," he says, staring at the floor. +His voice drops an octave, becomes flat. "THEY DON'T LEVITATE." + +Kapor chose Bust. He went back to the States and acquired a degree +in counselling psychology. He worked a while in a hospital, +couldn't stand that either. "My rep was," he says "a very bright kid +with a lot of potential who hasn't found himself. Almost thirty. +Sort of lost." + +Kapor was unemployed when he bought his first personal computer--an Apple II. +He sold his stereo to raise cash and drove to New Hampshire to avoid the +sales tax. + +"The day after I purchased it," Kapor tells me, "I was hanging out +in a computer store and I saw another guy, a man in his forties, +well-dressed guy, and eavesdropped on his conversation with the salesman. +He didn't know anything about computers. I'd had a year programming. +And I could program in BASIC. I'd taught myself. So I went up to him, +and I actually sold myself to him as a consultant." He pauses. +"I don't know where I got the nerve to do this. It was uncharacteristic. +I just said, `I think I can help you, I've been listening, +this is what you need to do and I think I can do it for you.' +And he took me on! He was my first client! I became a computer +consultant the first day after I bought the Apple II." + +Kapor had found his true vocation. He attracted more clients +for his consultant service, and started an Apple users' group. + +A friend of Kapor's, Eric Rosenfeld, a graduate student at MIT, +had a problem. He was doing a thesis on an arcane form of +financial statistics, but could not wedge himself into the crowded queue +for time on MIT's mainframes. (One might note at this point that if +Mr. Rosenfeld had dishonestly broken into the MIT mainframes, +Kapor himself might have never invented Lotus 1-2-3 and +the PC business might have been set back for years!) +Eric Rosenfeld did have an Apple II, however, +and he thought it might be possible to scale the problem down. +Kapor, as favor, wrote a program for him in BASIC that did the job. + +It then occurred to the two of them, out of the blue, +that it might be possible to SELL this program. +They marketed it themselves, in plastic baggies, +for about a hundred bucks a pop, mail order. +"This was a total cottage industry by a marginal consultant," +Kapor says proudly. "That's how I got started, honest to God." + +Rosenfeld, who later became a very prominent figure on Wall Street, +urged Kapor to go to MIT's business school for an MBA. +Kapor did seven months there, but never got his MBA. +He picked up some useful tools--mainly a firm grasp +of the principles of accounting--and, in his own words, +"learned to talk MBA." Then he dropped out and went to Silicon Valley. + +The inventors of VisiCalc, the Apple computer's premier business program, +had shown an interest in Mitch Kapor. Kapor worked diligently for them +for six months, got tired of California, and went back to Boston +where they had better bookstores. The VisiCalc group had made +the critical error of bringing in "professional management." +"That drove them into the ground," Kapor says. + +"Yeah, you don't hear a lot about VisiCalc these days," I muse. + +Kapor looks surprised. "Well, Lotus. . . we BOUGHT it." + +"Oh. You BOUGHT it?" + +"Yeah." + +"Sort of like the Bell System buying Western Union?" + +Kapor grins. "Yep! Yep! Yeah, exactly!" + +Mitch Kapor was not in full command of the destiny of himself +or his industry. The hottest software commodities of the early 1980s +were COMPUTER GAMES--the Atari seemed destined to enter every teenage home +in America. Kapor got into business software simply because he didn't have +any particular feeling for computer games. But he was supremely fast +on his feet, open to new ideas and inclined to trust his instincts. +And his instincts were good. He chose good people to deal with-- +gifted programmer Jonathan Sachs (the co-author of Lotus 1-2-3). +Financial wizard Eric Rosenfeld, canny Wall Street analyst +and venture capitalist Ben Rosen. Kapor was the founder and CEO of Lotus, +one of the most spectacularly successful business ventures of the +later twentieth century. + +He is now an extremely wealthy man. I ask him if he actually +knows how much money he has. + +"Yeah," he says. "Within a percent or two." + +How much does he actually have, then? + +He shakes his head. "A lot. A lot. Not something I talk about. +Issues of money and class are things that cut pretty close to the bone." + +I don't pry. It's beside the point. One might presume, impolitely, +that Kapor has at least forty million--that's what he got the year +he left Lotus. People who ought to know claim Kapor has about +a hundred and fifty million, give or take a market swing +in his stock holdings. If Kapor had stuck with Lotus, +as his colleague friend and rival Bill Gates has stuck +with his own software start-up, Microsoft, then Kapor +would likely have much the same fortune Gates has-- +somewhere in the neighborhood of three billion, +give or take a few hundred million. Mitch Kapor +has all the money he wants. Money has lost whatever charm +it ever held for him--probably not much in the first place. +When Lotus became too uptight, too bureaucratic, too far +from the true sources of his own satisfaction, Kapor walked. +He simply severed all connections with the company and went out the door. +It stunned everyone--except those who knew him best. + +Kapor has not had to strain his resources to wreak a thorough +transformation in cyberspace politics. In its first year, +EFF's budget was about a quarter of a million dollars. +Kapor is running EFF out of his pocket change. + +Kapor takes pains to tell me that he does not consider himself +a civil libertarian per se. He has spent quite some time +with true-blue civil libertarians lately, and there's a +political-correctness to them that bugs him. They seem +to him to spend entirely too much time in legal nitpicking +and not enough vigorously exercising civil rights in the +everyday real world. + +Kapor is an entrepreneur. Like all hackers, he prefers his involvements +direct, personal, and hands-on. "The fact that EFF has a node on the +Internet is a great thing. We're a publisher. We're a distributor +of information." Among the items the eff.org Internet node carries +is back issues of Phrack. They had an internal debate about that in EFF, +and finally decided to take the plunge. They might carry other +digital underground publications--but if they do, he says, +"we'll certainly carry Donn Parker, and anything Gail Thackeray +wants to put up. We'll turn it into a public library, that has +the whole spectrum of use. Evolve in the direction of people making up +their own minds." He grins. "We'll try to label all the editorials." + +Kapor is determined to tackle the technicalities of the Internet +in the service of the public interest. "The problem with being a node +on the Net today is that you've got to have a captive technical specialist. +We have Chris Davis around, for the care and feeding of the balky beast! +We couldn't do it ourselves!" + +He pauses. "So one direction in which technology has to evolve +is much more standardized units, that a non-technical person +can feel comfortable with. It's the same shift as from minicomputers to PCs. +I can see a future in which any person can have a Node on the Net. +Any person can be a publisher. It's better than the media we now have. +It's possible. We're working actively." + +Kapor is in his element now, fluent, thoroughly in command in his material. +"You go tell a hardware Internet hacker that everyone should have a node +on the Net," he says, "and the first thing they're going to say is, +`IP doesn't scale!'" ("IP" is the interface protocol for the Internet. +As it currently exists, the IP software is simply not capable of +indefinite expansion; it will run out of usable addresses, it will saturate.) +"The answer," Kapor says, "is: evolve the protocol! Get the smart people +together and figure out what to do. Do we add ID? Do we add new protocol? +Don't just say, WE CAN'T DO IT." + +Getting smart people together to figure out what to do is a skill +at which Kapor clearly excels. I counter that people on the Internet +rather enjoy their elite technical status, and don't seem particularly +anxious to democratize the Net. + +Kapor agrees, with a show of scorn. "I tell them that this is the snobbery +of the people on the Mayflower looking down their noses at the people +who came over ON THE SECOND BOAT! Just because they got here a year, +or five years, or ten years before everybody else, that doesn't give +them ownership of cyberspace! By what right?" + +I remark that the telcos are an electronic network, too, +and they seem to guard their specialized knowledge pretty closely. + +Kapor ripostes that the telcos and the Internet are entirely +different animals. "The Internet is an open system, +everything is published, everything gets argued about, +basically by anybody who can get in. Mostly, it's exclusive +and elitist just because it's so difficult. Let's make it easier to use." + +On the other hand, he allows with a swift change of emphasis, +the so-called elitists do have a point as well. "Before people start coming in, +who are new, who want to make suggestions, and criticize the Net as +`all screwed up'. . . . They should at least take the time to understand +the culture on its own terms. It has its own history--show some respect +for it. I'm a conservative, to that extent." + +The Internet is Kapor's paradigm for the future of telecommunications. +The Internet is decentralized, non-hierarchical, almost anarchic. +There are no bosses, no chain of command, no secret data. +If each node obeys the general interface standards, +there's simply no need for any central network authority. + +Wouldn't that spell the doom of AT&T as an institution? I ask. + +That prospect doesn't faze Kapor for a moment. "Their big advantage, +that they have now, is that they have all of the wiring. +But two things are happening. Anyone with right-of-way +is putting down fiber--Southern Pacific Railroad, +people like that--there's enormous `dark fiber' laid in." +("Dark Fiber" is fiber-optic cable, whose enormous capacity +so exceeds the demands of current usage that much of the +fiber still has no light-signals on it--it's still `dark,' +awaiting future use.) + +"The other thing that's happening is the local-loop stuff +is going to go wireless. Everyone from Bellcore to the cable TV +companies to AT&T wants to put in these things called +`personal communication systems.' So you could have local competition-- +you could have multiplicity of people, a bunch of neighborhoods, +sticking stuff up on poles. And a bunch of other people laying in dark fiber. +So what happens to the telephone companies? There's enormous pressure +on them from both sides. + +"The more I look at this, the more I believe that in a post-industrial, +digital world, the idea of regulated monopolies is bad. People will +look back on it and say that in the 19th and 20th centuries +the idea of public utilities was an okay compromise. +You needed one set of wires in the ground. It was too economically +inefficient, otherwise. And that meant one entity running it. +But now, with pieces being wireless--the connections are going +to be via high-level interfaces, not via wires. I mean, ULTIMATELY +there are going to be wires--but the wires are just a commodity. +Fiber, wireless. You no longer NEED a utility." + +Water utilities? Gas utilities? + +Of course we still need those, he agrees. "But when what you're moving +is information, instead of physical substances, then you can play by +a different set of rules. We're evolving those rules now! +Hopefully you can have a much more decentralized system, +and one in which there's more competition in the marketplace. + +"The role of government will be to make sure that nobody cheats. +The proverbial `level playing field.' A policy that prevents monopolization. +It should result in better service, lower prices, more choices, +and local empowerment." He smiles. "I'm very big on local empowerment." + +Kapor is a man with a vision. It's a very novel vision which he +and his allies are working out in considerable detail and with great energy. +Dark, cynical, morbid cyberpunk that I am, I cannot avoid considering +some of the darker implications of "decentralized, nonhierarchical, +locally empowered" networking. + +I remark that some pundits have suggested that electronic networking--faxes, +phones, small-scale photocopiers--played a strong role in dissolving +the power of centralized communism and causing the collapse of the Warsaw Pact. + +Socialism is totally discredited, says Kapor, fresh back from +the Eastern Bloc. The idea that faxes did it, all by themselves, +is rather wishful thinking. + +Has it occurred to him that electronic networking might corrode +America's industrial and political infrastructure to the point +where the whole thing becomes untenable, unworkable--and the old order +just collapses headlong, like in Eastern Europe? + +"No," Kapor says flatly. "I think that's extraordinarily unlikely. +In part, because ten or fifteen years ago, I had similar hopes +about personal computers--which utterly failed to materialize." +He grins wryly, then his eyes narrow. "I'm VERY opposed to techno-utopias. +Every time I see one, I either run away, or try to kill it." + +It dawns on me then that Mitch Kapor is not trying to +make the world safe for democracy. He certainly is not +trying to make it safe for anarchists or utopians-- +least of all for computer intruders or electronic rip-off artists. +What he really hopes to do is make the world safe for +future Mitch Kapors. This world of decentralized, small-scale nodes, +with instant global access for the best and brightest, +would be a perfect milieu for the shoestring attic capitalism +that made Mitch Kapor what he is today. + +Kapor is a very bright man. He has a rare combination +of visionary intensity with a strong practical streak. +The Board of the EFF: John Barlow, Jerry Berman of the ACLU, +Stewart Brand, John Gilmore, Steve Wozniak, and Esther Dyson, +the doyenne of East-West computer entrepreneurism--share his gift, +his vision, and his formidable networking talents. +They are people of the 1960s, winnowed-out by its turbulence +and rewarded with wealth and influence. They are some of the best +and the brightest that the electronic community has to offer. +But can they do it, in the real world? Or are they only dreaming? +They are so few. And there is so much against them. + +I leave Kapor and his networking employees struggling cheerfully +with the promising intricacies of their newly installed Macintosh +System 7 software. The next day is Saturday. EFF is closed. +I pay a few visits to points of interest downtown. + +One of them is the birthplace of the telephone. + +It's marked by a bronze plaque in a plinth of black-and-white speckled granite. It sits in the +plaza of the John F. Kennedy Federal Building, the very place where Kapor was +once fingerprinted by the FBI. + +The plaque has a bas-relief picture of Bell's original telephone. +"BIRTHPLACE OF THE TELEPHONE," it reads. "Here, on June 2, 1875, +Alexander Graham Bell and Thomas A. Watson first transmitted sound over wires. + +"This successful experiment was completed in a fifth floor garret +at what was then 109 Court Street and marked the beginning of +world-wide telephone service." + +109 Court Street is long gone. Within sight of Bell's plaque, +across a street, is one of the central offices of NYNEX, +the local Bell RBOC, on 6 Bowdoin Square. + +I cross the street and circle the telco building, slowly, +hands in my jacket pockets. It's a bright, windy, New England +autumn day. The central office is a handsome 1940s-era megalith +in late Art Deco, eight stories high. + +Parked outside the back is a power-generation truck. +The generator strikes me as rather anomalous. Don't they +already have their own generators in this eight-story monster? +Then the suspicion strikes me that NYNEX must have heard +of the September 17 AT&T power-outage which crashed New York City. +Belt-and-suspenders, this generator. Very telco. + +Over the glass doors of the front entrance is a handsome bronze +bas-relief of Art Deco vines, sunflowers, and birds, entwining +the Bell logo and the legend NEW ENGLAND TELEPHONE AND TELEGRAPH COMPANY +--an entity which no longer officially exists. + +The doors are locked securely. I peer through the shadowed glass. +Inside is an official poster reading: + + +"New England Telephone a NYNEX Company + +ATTENTION + +"All persons while on New England Telephone +Company premises are required to visibly wear their +identification cards (C.C.P. Section 2, Page 1). + +"Visitors, vendors, contractors, and all others are +required to visibly wear a daily pass. + +"Thank you. + +Kevin C. Stanton. +Building Security Coordinator." + + +Outside, around the corner, is a pull-down ribbed metal security door, +a locked delivery entrance. Some passing stranger has grafitti-tagged +this door, with a single word in red spray-painted cursive: + +Fury + +# + +My book on the Hacker Crackdown is almost over now. +I have deliberately saved the best for last. + +In February 1991, I attended the CPSR Public Policy Roundtable, +in Washington, DC. CPSR, Computer Professionals for Social Responsibility, +was a sister organization of EFF, or perhaps its aunt, being older +and perhaps somewhat wiser in the ways of the world of politics. + +Computer Professionals for Social Responsibility began in 1981 +in Palo Alto, as an informal discussion group of Californian +computer scientists and technicians, united by nothing more +than an electronic mailing list. This typical high-tech +ad-hocracy received the dignity of its own acronym in 1982, +and was formally incorporated in 1983. + +CPSR lobbied government and public alike with an educational +outreach effort, sternly warning against any foolish +and unthinking trust in complex computer systems. +CPSR insisted that mere computers should never be +considered a magic panacea for humanity's social, +ethical or political problems. CPSR members were especially +troubled about the stability, safety, and dependability +of military computer systems, and very especially troubled +by those systems controlling nuclear arsenals. CPSR was +best-known for its persistent and well-publicized attacks on the +scientific credibility of the Strategic Defense Initiative ("Star Wars"). + +In 1990, CPSR was the nation's veteran cyber-political activist group, +with over two thousand members in twenty- one local chapters across the US. +It was especially active in Boston, Silicon Valley, and Washington DC, +where its Washington office sponsored the Public Policy Roundtable. + +The Roundtable, however, had been funded by EFF, which had passed CPSR +an extensive grant for operations. This was the first large-scale, +official meeting of what was to become the electronic civil +libertarian community. + +Sixty people attended, myself included--in this instance, not so much +as a journalist as a cyberpunk author. Many of the luminaries +of the field took part: Kapor and Godwin as a matter of course. +Richard Civille and Marc Rotenberg of CPSR. Jerry Berman of the ACLU. +John Quarterman, author of The Matrix. Steven Levy, author of Hackers. +George Perry and Sandy Weiss of Prodigy Services, there to network +about the civil-liberties troubles their young commercial +network was experiencing. Dr. Dorothy Denning. Cliff Figallo, +manager of the Well. Steve Jackson was there, having finally +found his ideal target audience, and so was Craig Neidorf, +"Knight Lightning" himself, with his attorney, Sheldon Zenner. +Katie Hafner, science journalist, and co-author of Cyberpunk: +Outlaws and Hackers on the Computer Frontier. Dave Farber, +ARPAnet pioneer and fabled Internet guru. Janlori Goldman +of the ACLU's Project on Privacy and Technology. John Nagle +of Autodesk and the Well. Don Goldberg of the House Judiciary Committee. +Tom Guidoboni, the defense attorney in the Internet Worm case. +Lance Hoffman, computer-science professor at The George Washington +University. Eli Noam of Columbia. And a host of others no less distinguished. + +Senator Patrick Leahy delivered the keynote address, +expressing his determination to keep ahead of the curve +on the issue of electronic free speech. The address was +well-received, and the sense of excitement was palpable. +Every panel discussion was interesting--some were entirely +compelling. People networked with an almost frantic interest. + +I myself had a most interesting and cordial lunch discussion with +Noel and Jeanne Gayler, Admiral Gayler being a former director +of the National Security Agency. As this was the first known encounter +between an actual no-kidding cyberpunk and a chief executive of +America's largest and best-financed electronic espionage apparat, +there was naturally a bit of eyebrow-raising on both sides. + +Unfortunately, our discussion was off-the-record. In fact +all the discussions at the CPSR were officially off-the-record, +the idea being to do some serious networking in an atmosphere +of complete frankness, rather than to stage a media circus. + +In any case, CPSR Roundtable, though interesting and intensely valuable, +was as nothing compared to the truly mind-boggling event that transpired +a mere month later. + +# + +"Computers, Freedom and Privacy." Four hundred people from +every conceivable corner of America's electronic community. +As a science fiction writer, I have been to some weird gigs in my day, +but this thing is truly BEYOND THE PALE. Even "Cyberthon," +Point Foundation's "Woodstock of Cyberspace" where Bay Area +psychedelia collided headlong with the emergent world +of computerized virtual reality, was like a Kiwanis Club gig +compared to this astonishing do. + +The "electronic community" had reached an apogee. +Almost every principal in this book is in attendance. +Civil Libertarians. Computer Cops. The Digital Underground. +Even a few discreet telco people. Colorcoded dots +for lapel tags are distributed. Free Expression issues. +Law Enforcement. Computer Security. Privacy. Journalists. +Lawyers. Educators. Librarians. Programmers. +Stylish punk-black dots for the hackers and phone phreaks. +Almost everyone here seems to wear eight or nine dots, +to have six or seven professional hats. + +It is a community. Something like Lebanon perhaps, +but a digital nation. People who had feuded all year +in the national press, people who entertained the deepest +suspicions of one another's motives and ethics, are now +in each others' laps. "Computers, Freedom and Privacy" +had every reason in the world to turn ugly, and yet except +for small irruptions of puzzling nonsense from the +convention's token lunatic, a surprising bonhomie reigned. +CFP was like a wedding-party in which two lovers, +unstable bride and charlatan groom, tie the knot +in a clearly disastrous matrimony. + +It is clear to both families--even to neighbors and random guests-- +that this is not a workable relationship, and yet the young couple's +desperate attraction can brook no further delay. They simply cannot +help themselves. Crockery will fly, shrieks from their newlywed home +will wake the city block, divorce waits in the wings like a vulture +over the Kalahari, and yet this is a wedding, and there is going +to be a child from it. Tragedies end in death; comedies in marriage. +The Hacker Crackdown is ending in marriage. And there will be a child. + +From the beginning, anomalies reign. John Perry Barlow, +cyberspace ranger, is here. His color photo in +The New York Times Magazine, Barlow scowling +in a grim Wyoming snowscape, with long black coat, +dark hat, a Macintosh SE30 propped on a fencepost +and an awesome frontier rifle tucked under one arm, +will be the single most striking visual image +of the Hacker Crackdown. And he is CFP's guest of honor-- +along with Gail Thackeray of the FCIC! What on earth do +they expect these dual guests to do with each other? Waltz? + +Barlow delivers the first address. Uncharacteristically, +he is hoarse--the sheer volume of roadwork has worn him down. +He speaks briefly, congenially, in a plea for conciliation, +and takes his leave to a storm of applause. + +Then Gail Thackeray takes the stage. She's visibly nervous. +She's been on the Well a lot lately. Reading those Barlow posts. +Following Barlow is a challenge to anyone. In honor of the famous +lyricist for the Grateful Dead, she announces reedily, she is going to read-- +A POEM. A poem she has composed herself. + +It's an awful poem, doggerel in the rollicking meter of Robert W. Service's +The Cremation of Sam McGee, but it is in fact, a poem. It's the Ballad +of the Electronic Frontier! A poem about the Hacker Crackdown and the +sheer unlikelihood of CFP. It's full of in-jokes. The score or so cops +in the audience, who are sitting together in a nervous claque, +are absolutely cracking-up. Gail's poem is the funniest goddamn thing +they've ever heard. The hackers and civil-libs, who had this woman figured +for Ilsa She-Wolf of the SS, are staring with their jaws hanging loosely. +Never in the wildest reaches of their imagination had they figured +Gail Thackeray was capable of such a totally off-the-wall move. +You can see them punching their mental CONTROL-RESET buttons. +Jesus! This woman's a hacker weirdo! She's JUST LIKE US! +God, this changes everything! + +Al Bayse, computer technician for the FBI, had been the only cop +at the CPSR Roundtable, dragged there with his arm bent by +Dorothy Denning. He was guarded and tightlipped at CPSR Roundtable; +a "lion thrown to the Christians." + +At CFP, backed by a claque of cops, Bayse suddenly waxes eloquent +and even droll, describing the FBI's "NCIC 2000", a gigantic digital catalog +of criminal records, as if he has suddenly become some weird hybrid +of George Orwell and George Gobel. Tentatively, he makes an arcane +joke about statistical analysis. At least a third of the crowd laughs aloud. + +"They didn't laugh at that at my last speech," Bayse observes. +He had been addressing cops--STRAIGHT cops, not computer people. +It had been a worthy meeting, useful one supposes, but nothing like THIS. +There has never been ANYTHING like this. Without any prodding, +without any preparation, people in the audience simply begin to ask questions. +Longhairs, freaky people, mathematicians. Bayse is answering, politely, +frankly, fully, like a man walking on air. The ballroom's atmosphere +crackles with surreality. A female lawyer behind me breaks into a sweat +and a hot waft of surprisingly potent and musky perfume flows off +her pulse-points. + +People are giddy with laughter. People are interested, +fascinated, their eyes so wide and dark that they seem eroticized. +Unlikely daisy-chains form in the halls, around the bar, on the escalators: +cops with hackers, civil rights with FBI, Secret Service with phone phreaks. + +Gail Thackeray is at her crispest in a white wool sweater with a +tiny Secret Service logo. "I found Phiber Optik at the payphones, +and when he saw my sweater, he turned into a PILLAR OF SALT!" she chortles. + +Phiber discusses his case at much length with his arresting officer, +Don Delaney of the New York State Police. After an hour's chat, +the two of them look ready to begin singing "Auld Lang Syne." +Phiber finally finds the courage to get his worst complaint off his chest. +It isn't so much the arrest. It was the CHARGE. Pirating service +off 900 numbers. I'm a PROGRAMMER, Phiber insists. This lame charge +is going to hurt my reputation. It would have been cool to be busted +for something happening, like Section 1030 computer intrusion. +Maybe some kind of crime that's scarcely been invented yet. +Not lousy phone fraud. Phooey. + +Delaney seems regretful. He had a mountain of possible criminal charges +against Phiber Optik. The kid's gonna plead guilty anyway. He's a +first timer, they always plead. Coulda charged the kid with most anything, +and gotten the same result in the end. Delaney seems genuinely sorry +not to have gratified Phiber in this harmless fashion. Too late now. +Phiber's pled already. All water under the bridge. Whaddya gonna do? + +Delaney's got a good grasp on the hacker mentality. +He held a press conference after he busted a bunch of +Masters of Deception kids. Some journo had asked him: +"Would you describe these people as GENIUSES?" +Delaney's deadpan answer, perfect: "No, I would describe +these people as DEFENDANTS." Delaney busts a kid for +hacking codes with repeated random dialling. Tells the +press that NYNEX can track this stuff in no time flat nowadays, +and a kid has to be STUPID to do something so easy to catch. +Dead on again: hackers don't mind being thought of as Genghis Khan +by the straights, but if there's anything that really gets 'em +where they live, it's being called DUMB. + +Won't be as much fun for Phiber next time around. +As a second offender he's gonna see prison. +Hackers break the law. They're not geniuses, either. +They're gonna be defendants. And yet, Delaney muses over +a drink in the hotel bar, he has found it impossible to treat +them as common criminals. Delaney knows criminals. These kids, +by comparison, are clueless--there is just no crook vibe off of them, +they don't smell right, they're just not BAD. + +Delaney has seen a lot of action. He did Vietnam. +He's been shot at, he has shot people. He's a homicide +cop from New York. He has the appearance of a man who +has not only seen the shit hit the fan but has seen it splattered +across whole city blocks and left to ferment for years. +This guy has been around. + +He listens to Steve Jackson tell his story. The dreamy +game strategist has been dealt a bad hand. He has played +it for all he is worth. Under his nerdish SF-fan exterior +is a core of iron. Friends of his say Steve Jackson believes +in the rules, believes in fair play. He will never compromise +his principles, never give up. "Steve," Delaney says to +Steve Jackson, "they had some balls, whoever busted you. +You're all right!" Jackson, stunned, falls silent and +actually blushes with pleasure. + +Neidorf has grown up a lot in the past year. The kid is +a quick study, you gotta give him that. Dressed by his mom, +the fashion manager for a national clothing chain, +Missouri college techie-frat Craig Neidorf out-dappers +everyone at this gig but the toniest East Coast lawyers. +The iron jaws of prison clanged shut without him and now +law school beckons for Neidorf. He looks like a larval Congressman. + +Not a "hacker," our Mr. Neidorf. He's not interested +in computer science. Why should he be? He's not +interested in writing C code the rest of his life, +and besides, he's seen where the chips fall. +To the world of computer science he and Phrack +were just a curiosity. But to the world of law. . . . +The kid has learned where the bodies are buried. +He carries his notebook of press clippings wherever he goes. + +Phiber Optik makes fun of Neidorf for a Midwestern geek, +for believing that "Acid Phreak" does acid and listens to acid rock. +Hell no. Acid's never done ACID! Acid's into ACID HOUSE MUSIC. +Jesus. The very idea of doing LSD. Our PARENTS did LSD, ya clown. + +Thackeray suddenly turns upon Craig Neidorf the full lighthouse +glare of her attention and begins a determined half-hour attempt +to WIN THE BOY OVER. The Joan of Arc of Computer Crime is +GIVING CAREER ADVICE TO KNIGHT LIGHTNING! "Your experience +would be very valuable--a real asset," she tells him with +unmistakeable sixty-thousand-watt sincerity. Neidorf is fascinated. +He listens with unfeigned attention. He's nodding and saying yes ma'am. +Yes, Craig, you too can forget all about money and enter the glamorous +and horribly underpaid world of PROSECUTING COMPUTER CRIME! +You can put your former friends in prison--ooops. . . . + +You cannot go on dueling at modem's length indefinitely. +You cannot beat one another senseless with rolled-up press-clippings. +Sooner or later you have to come directly to grips. +And yet the very act of assembling here has changed +the entire situation drastically. John Quarterman, +author of The Matrix, explains the Internet at his symposium. +It is the largest news network in the world, it is growing +by leaps and bounds, and yet you cannot measure Internet because +you cannot stop it in place. It cannot stop, because there +is no one anywhere in the world with the authority to stop Internet. +It changes, yes, it grows, it embeds itself across the post-industrial, +postmodern world and it generates community wherever it +touches, and it is doing this all by itself. + +Phiber is different. A very fin de siecle kid, Phiber Optik. +Barlow says he looks like an Edwardian dandy. He does rather. +Shaven neck, the sides of his skull cropped hip-hop close, +unruly tangle of black hair on top that looks pomaded, +he stays up till four a.m. and misses all the sessions, +then hangs out in payphone booths with his acoustic coupler +gutsily CRACKING SYSTEMS RIGHT IN THE MIDST OF THE HEAVIEST +LAW ENFORCEMENT DUDES IN THE U.S., or at least PRETENDING to. . . . +Unlike "Frank Drake." Drake, who wrote Dorothy Denning out +of nowhere, and asked for an interview for his cheapo +cyberpunk fanzine, and then started grilling her on her ethics. +She was squirmin', too. . . . Drake, scarecrow-tall with his +floppy blond mohawk, rotting tennis shoes and black leather jacket +lettered ILLUMINATI in red, gives off an unmistakeable air +of the bohemian literatus. Drake is the kind of guy +who reads British industrial design magazines and appreciates +William Gibson because the quality of the prose is so tasty. +Drake could never touch a phone or a keyboard again, +and he'd still have the nose-ring and the blurry photocopied +fanzines and the sampled industrial music. He's a radical punk +with a desktop-publishing rig and an Internet address. +Standing next to Drake, the diminutive Phiber looks like he's +been physically coagulated out of phone-lines. Born to phreak. + +Dorothy Denning approaches Phiber suddenly. The two of them +are about the same height and body-build. Denning's blue eyes +flash behind the round window-frames of her glasses. +"Why did you say I was `quaint?'" she asks Phiber, quaintly. + +It's a perfect description but Phiber is nonplussed. . . +"Well, I uh, you know. . . ." + +"I also think you're quaint, Dorothy," I say, novelist to the rescue, +the journo gift of gab. . . . She is neat and dapper and yet there's +an arcane quality to her, something like a Pilgrim Maiden behind +leaded glass; if she were six inches high Dorothy Denning would look +great inside a china cabinet. . .The Cryptographeress. . . +The Cryptographrix. . .whatever. . . . Weirdly, Peter Denning looks +just like his wife, you could pick this gentleman out of a thousand guys +as the soulmate of Dorothy Denning. Wearing tailored slacks, +a spotless fuzzy varsity sweater, and a neatly knotted academician's tie. . . . +This fineboned, exquisitely polite, utterly civilized and hyperintelligent +couple seem to have emerged from some cleaner and finer parallel universe, +where humanity exists to do the Brain Teasers column in Scientific American. +Why does this Nice Lady hang out with these unsavory characters? + +Because the time has come for it, that's why. +Because she's the best there is at what she does. + +Donn Parker is here, the Great Bald Eagle of Computer Crime. . . . +With his bald dome, great height, and enormous Lincoln-like hands, +the great visionary pioneer of the field plows through the lesser mortals +like an icebreaker. . . . His eyes are fixed on the future with the +rigidity of a bronze statue. . . . Eventually, he tells his audience, +all business crime will be computer crime, because businesses will do +everything through computers. "Computer crime" as a category will vanish. + +In the meantime, passing fads will flourish and fail and evaporate. . . . +Parker's commanding, resonant voice is sphinxlike, everything is viewed +from some eldritch valley of deep historical abstraction. . . . +Yes, they've come and they've gone, these passing flaps in the world +of digital computation. . . . The radio-frequency emanation scandal. . . +KGB and MI5 and CIA do it every day, it's easy, but nobody else ever has. . . . +The salami-slice fraud, mostly mythical. . . . "Crimoids," he calls them. . . . +Computer viruses are the current crimoid champ, a lot less dangerous than +most people let on, but the novelty is fading and there's a crimoid vacuum at +the moment, the press is visibly hungering for something more outrageous. . . . +The Great Man shares with us a few speculations on the coming crimoids. . . . +Desktop Forgery! Wow. . . . Computers stolen just for the sake of the +information within them--data-napping! Happened in Britain a while ago, +could be the coming thing. . . . Phantom nodes in the Internet! + +Parker handles his overhead projector sheets with an ecclesiastical air. . . . +He wears a grey double-breasted suit, a light blue shirt, and a +very quiet tie of understated maroon and blue paisley. . . . +Aphorisms emerge from him with slow, leaden emphasis. . . . +There is no such thing as an adequately secure computer +when one faces a sufficiently powerful adversary. . . . +Deterrence is the most socially useful aspect of security. . . . +People are the primary weakness in all information systems. . . . +The entire baseline of computer security must be shifted upward. . . . +Don't ever violate your security by publicly describing +your security measures. . . . + +People in the audience are beginning to squirm, and yet +there is something about the elemental purity of this guy's +philosophy that compels uneasy respect. . . . Parker sounds +like the only sane guy left in the lifeboat, sometimes. +The guy who can prove rigorously, from deep moral principles, +that Harvey there, the one with the broken leg and the checkered past, +is the one who has to be, err. . .that is, Mr. Harvey is best placed +to make the necessary sacrifice for the security and indeed +the very survival of the rest of this lifeboat's crew. . . . +Computer security, Parker informs us mournfully, is a +nasty topic, and we wish we didn't have to have it. . . . +The security expert, armed with method and logic, must think--imagine-- +everything that the adversary might do before the adversary might +actually do it. It is as if the criminal's dark brain were an +extensive subprogram within the shining cranium of Donn Parker. +He is a Holmes whose Moriarty does not quite yet exist +and so must be perfectly simulated. + +CFP is a stellar gathering, with the giddiness of a wedding. +It is a happy time, a happy ending, they know their world +is changing forever tonight, and they're proud to have been there +to see it happen, to talk, to think, to help. + +And yet as night falls, a certain elegiac quality manifests itself, +as the crowd gathers beneath the chandeliers with their wineglasses +and dessert plates. Something is ending here, gone forever, +and it takes a while to pinpoint it. + +It is the End of the Amateurs. + + + + + + + + + +End of the Project Gutenberg EBook of Hacker Crackdown, by Bruce Sterling + +*** END OF THIS PROJECT GUTENBERG EBOOK HACKER CRACKDOWN *** + +***** This file should be named 101.txt or 101.zip ***** +This and all associated files of various formats will be found in: + http://www.gutenberg.org/1/0/101/ + + + +Updated editions will replace the previous one--the old editions will be +renamed. + +Creating the works from public domain print editions means that no one +owns a United States copyright in these works, so the Foundation (and +you!) can copy and distribute it in the United States without permission +and without paying copyright royalties. Special rules, set forth in the +General Terms of Use part of this license, apply to copying and +distributing Project Gutenberg-tm electronic works to protect the +PROJECT GUTENBERG-tm concept and trademark. Project Gutenberg is a +registered trademark, and may not be used if you charge for the eBooks, +unless you receive specific permission. If you do not charge anything +for copies of this eBook, complying with the rules is very easy. You may +use this eBook for nearly any purpose such as creation of derivative +works, reports, performances and research. They may be modified and +printed and given away--you may do practically ANYTHING with public +domain eBooks. Redistribution is subject to the trademark license, +especially commercial redistribution. + + + +*** START: FULL LICENSE *** + +THE FULL PROJECT GUTENBERG LICENSE +PLEASE READ THIS BEFORE YOU DISTRIBUTE OR USE THIS WORK + +To protect the Project Gutenberg-tm mission of promoting the free +distribution of electronic works, by using or distributing this work +(or any other work associated in any way with the phrase "Project +Gutenberg"), you agree to comply with all the terms of the Full Project +Gutenberg-tm License (available with this file or online at +http://www.gutenberg.org/license). + + +Section 1. General Terms of Use and Redistributing Project Gutenberg-tm +electronic works + +1.A. By reading or using any part of this Project Gutenberg-tm +electronic work, you indicate that you have read, understand, agree to +and accept all the terms of this license and intellectual property +(trademark/copyright) agreement. If you do not agree to abide by all +the terms of this agreement, you must cease using and return or destroy +all copies of Project Gutenberg-tm electronic works in your possession. +If you paid a fee for obtaining a copy of or access to a Project +Gutenberg-tm electronic work and you do not agree to be bound by the +terms of this agreement, you may obtain a refund from the person or +entity to whom you paid the fee as set forth in paragraph 1.E.8. + +1.B. "Project Gutenberg" is a registered trademark. It may only be +used on or associated in any way with an electronic work by people who +agree to be bound by the terms of this agreement. There are a few +things that you can do with most Project Gutenberg-tm electronic works +even without complying with the full terms of this agreement. See +paragraph 1.C below. There are a lot of things you can do with Project +Gutenberg-tm electronic works if you follow the terms of this agreement +and help preserve free future access to Project Gutenberg-tm electronic +works. See paragraph 1.E below. + +1.C. The Project Gutenberg Literary Archive Foundation ("the Foundation" +or PGLAF), owns a compilation copyright in the collection of Project +Gutenberg-tm electronic works. Nearly all the individual works in the +collection are in the public domain in the United States. If an +individual work is in the public domain in the United States and you are +located in the United States, we do not claim a right to prevent you from +copying, distributing, performing, displaying or creating derivative +works based on the work as long as all references to Project Gutenberg +are removed. Of course, we hope that you will support the Project +Gutenberg-tm mission of promoting free access to electronic works by +freely sharing Project Gutenberg-tm works in compliance with the terms of +this agreement for keeping the Project Gutenberg-tm name associated with +the work. You can easily comply with the terms of this agreement by +keeping this work in the same format with its attached full Project +Gutenberg-tm License when you share it without charge with others. +This particular work is one of the few copyrighted individual works +included with the permission of the copyright holder. Information on +the copyright owner for this particular work and the terms of use +imposed by the copyright holder on this work are set forth at the +beginning of this work. + +1.D. The copyright laws of the place where you are located also govern +what you can do with this work. Copyright laws in most countries are in +a constant state of change. If you are outside the United States, check +the laws of your country in addition to the terms of this agreement +before downloading, copying, displaying, performing, distributing or +creating derivative works based on this work or any other Project +Gutenberg-tm work. The Foundation makes no representations concerning +the copyright status of any work in any country outside the United +States. + +1.E. Unless you have removed all references to Project Gutenberg: + +1.E.1. The following sentence, with active links to, or other immediate +access to, the full Project Gutenberg-tm License must appear prominently +whenever any copy of a Project Gutenberg-tm work (any work on which the +phrase "Project Gutenberg" appears, or with which the phrase "Project +Gutenberg" is associated) is accessed, displayed, performed, viewed, +copied or distributed: + +This eBook is for the use of anyone anywhere at no cost and with +almost no restrictions whatsoever. You may copy it, give it away or +re-use it under the terms of the Project Gutenberg License included +with this eBook or online at www.gutenberg.org + +1.E.2. If an individual Project Gutenberg-tm electronic work is derived +from the public domain (does not contain a notice indicating that it is +posted with permission of the copyright holder), the work can be copied +and distributed to anyone in the United States without paying any fees +or charges. If you are redistributing or providing access to a work +with the phrase "Project Gutenberg" associated with or appearing on the +work, you must comply either with the requirements of paragraphs 1.E.1 +through 1.E.7 or obtain permission for the use of the work and the +Project Gutenberg-tm trademark as set forth in paragraphs 1.E.8 or +1.E.9. + +1.E.3. If an individual Project Gutenberg-tm electronic work is posted +with the permission of the copyright holder, your use and distribution +must comply with both paragraphs 1.E.1 through 1.E.7 and any additional +terms imposed by the copyright holder. Additional terms will be linked +to the Project Gutenberg-tm License for all works posted with the +permission of the copyright holder found at the beginning of this work. + +1.E.4. Do not unlink or detach or remove the full Project Gutenberg-tm +License terms from this work, or any files containing a part of this +work or any other work associated with Project Gutenberg-tm. + +1.E.5. Do not copy, display, perform, distribute or redistribute this +electronic work, or any part of this electronic work, without +prominently displaying the sentence set forth in paragraph 1.E.1 with +active links or immediate access to the full terms of the Project +Gutenberg-tm License. + +1.E.6. You may convert to and distribute this work in any binary, +compressed, marked up, nonproprietary or proprietary form, including any +word processing or hypertext form. However, if you provide access to or +distribute copies of a Project Gutenberg-tm work in a format other than +"Plain Vanilla ASCII" or other format used in the official version +posted on the official Project Gutenberg-tm web site (www.gutenberg.org), +you must, at no additional cost, fee or expense to the user, provide a +copy, a means of exporting a copy, or a means of obtaining a copy upon +request, of the work in its original "Plain Vanilla ASCII" or other +form. Any alternate format must include the full Project Gutenberg-tm +License as specified in paragraph 1.E.1. + +1.E.7. Do not charge a fee for access to, viewing, displaying, +performing, copying or distributing any Project Gutenberg-tm works +unless you comply with paragraph 1.E.8 or 1.E.9. + +1.E.8. You may charge a reasonable fee for copies of or providing +access to or distributing Project Gutenberg-tm electronic works provided +that + +- You pay a royalty fee of 20% of the gross profits you derive from + the use of Project Gutenberg-tm works calculated using the method + you already use to calculate your applicable taxes. The fee is + owed to the owner of the Project Gutenberg-tm trademark, but he + has agreed to donate royalties under this paragraph to the + Project Gutenberg Literary Archive Foundation. Royalty payments + must be paid within 60 days following each date on which you + prepare (or are legally required to prepare) your periodic tax + returns. Royalty payments should be clearly marked as such and + sent to the Project Gutenberg Literary Archive Foundation at the + address specified in Section 4, "Information about donations to + the Project Gutenberg Literary Archive Foundation." + +- You provide a full refund of any money paid by a user who notifies + you in writing (or by e-mail) within 30 days of receipt that s/he + does not agree to the terms of the full Project Gutenberg-tm + License. You must require such a user to return or + destroy all copies of the works possessed in a physical medium + and discontinue all use of and all access to other copies of + Project Gutenberg-tm works. + +- You provide, in accordance with paragraph 1.F.3, a full refund of any + money paid for a work or a replacement copy, if a defect in the + electronic work is discovered and reported to you within 90 days + of receipt of the work. + +- You comply with all other terms of this agreement for free + distribution of Project Gutenberg-tm works. + +1.E.9. If you wish to charge a fee or distribute a Project Gutenberg-tm +electronic work or group of works on different terms than are set +forth in this agreement, you must obtain permission in writing from +both the Project Gutenberg Literary Archive Foundation and Michael +Hart, the owner of the Project Gutenberg-tm trademark. Contact the +Foundation as set forth in Section 3 below. + +1.F. + +1.F.1. Project Gutenberg volunteers and employees expend considerable +effort to identify, do copyright research on, transcribe and proofread +public domain works in creating the Project Gutenberg-tm +collection. Despite these efforts, Project Gutenberg-tm electronic +works, and the medium on which they may be stored, may contain +"Defects," such as, but not limited to, incomplete, inaccurate or +corrupt data, transcription errors, a copyright or other intellectual +property infringement, a defective or damaged disk or other medium, a +computer virus, or computer codes that damage or cannot be read by +your equipment. + +1.F.2. LIMITED WARRANTY, DISCLAIMER OF DAMAGES - Except for the "Right +of Replacement or Refund" described in paragraph 1.F.3, the Project +Gutenberg Literary Archive Foundation, the owner of the Project +Gutenberg-tm trademark, and any other party distributing a Project +Gutenberg-tm electronic work under this agreement, disclaim all +liability to you for damages, costs and expenses, including legal +fees. YOU AGREE THAT YOU HAVE NO REMEDIES FOR NEGLIGENCE, STRICT +LIABILITY, BREACH OF WARRANTY OR BREACH OF CONTRACT EXCEPT THOSE +PROVIDED IN PARAGRAPH 1.F.3. YOU AGREE THAT THE FOUNDATION, THE +TRADEMARK OWNER, AND ANY DISTRIBUTOR UNDER THIS AGREEMENT WILL NOT BE +LIABLE TO YOU FOR ACTUAL, DIRECT, INDIRECT, CONSEQUENTIAL, PUNITIVE OR +INCIDENTAL DAMAGES EVEN IF YOU GIVE NOTICE OF THE POSSIBILITY OF SUCH +DAMAGE. + +1.F.3. LIMITED RIGHT OF REPLACEMENT OR REFUND - If you discover a +defect in this electronic work within 90 days of receiving it, you can +receive a refund of the money (if any) you paid for it by sending a +written explanation to the person you received the work from. If you +received the work on a physical medium, you must return the medium with +your written explanation. The person or entity that provided you with +the defective work may elect to provide a replacement copy in lieu of a +refund. If you received the work electronically, the person or entity +providing it to you may choose to give you a second opportunity to +receive the work electronically in lieu of a refund. If the second copy +is also defective, you may demand a refund in writing without further +opportunities to fix the problem. + +1.F.4. Except for the limited right of replacement or refund set forth +in paragraph 1.F.3, this work is provided to you 'AS-IS,' WITH NO OTHER +WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +WARRANTIES OF MERCHANTIBILITY OR FITNESS FOR ANY PURPOSE. + +1.F.5. Some states do not allow disclaimers of certain implied +warranties or the exclusion or limitation of certain types of damages. +If any disclaimer or limitation set forth in this agreement violates the +law of the state applicable to this agreement, the agreement shall be +interpreted to make the maximum disclaimer or limitation permitted by +the applicable state law. The invalidity or unenforceability of any +provision of this agreement shall not void the remaining provisions. + +1.F.6. INDEMNITY - You agree to indemnify and hold the Foundation, the +trademark owner, any agent or employee of the Foundation, anyone +providing copies of Project Gutenberg-tm electronic works in accordance +with this agreement, and any volunteers associated with the production, +promotion and distribution of Project Gutenberg-tm electronic works, +harmless from all liability, costs and expenses, including legal fees, +that arise directly or indirectly from any of the following which you do +or cause to occur: (a) distribution of this or any Project Gutenberg-tm +work, (b) alteration, modification, or additions or deletions to any +Project Gutenberg-tm work, and (c) any Defect you cause. + + +Section 2. Information about the Mission of Project Gutenberg-tm + +Project Gutenberg-tm is synonymous with the free distribution of +electronic works in formats readable by the widest variety of computers +including obsolete, old, middle-aged and new computers. It exists +because of the efforts of hundreds of volunteers and donations from +people in all walks of life. + +Volunteers and financial support to provide volunteers with the +assistance they need are critical to reaching Project Gutenberg-tm's +goals and ensuring that the Project Gutenberg-tm collection will +remain freely available for generations to come. In 2001, the Project +Gutenberg Literary Archive Foundation was created to provide a secure +and permanent future for Project Gutenberg-tm and future generations. +To learn more about the Project Gutenberg Literary Archive Foundation +and how your efforts and donations can help, see Sections 3 and 4 +and the Foundation web page at http://www.pglaf.org. + + +Section 3. Information about the Project Gutenberg Literary Archive +Foundation + +The Project Gutenberg Literary Archive Foundation is a non profit +501(c)(3) educational corporation organized under the laws of the +state of Mississippi and granted tax exempt status by the Internal +Revenue Service. The Foundation's EIN or federal tax identification +number is 64-6221541. Its 501(c)(3) letter is posted at +http://pglaf.org/fundraising. Contributions to the Project Gutenberg +Literary Archive Foundation are tax deductible to the full extent +permitted by U.S. federal laws and your state's laws. + +The Foundation's principal office is located at 4557 Melan Dr. S. +Fairbanks, AK, 99712., but its volunteers and employees are scattered +throughout numerous locations. Its business office is located at +809 North 1500 West, Salt Lake City, UT 84116, (801) 596-1887, email +business@pglaf.org. Email contact links and up to date contact +information can be found at the Foundation's web site and official +page at http://pglaf.org + +For additional contact information: + Dr. Gregory B. Newby + Chief Executive and Director + gbnewby@pglaf.org + +Section 4. Information about Donations to the Project Gutenberg +Literary Archive Foundation + +Project Gutenberg-tm depends upon and cannot survive without wide +spread public support and donations to carry out its mission of +increasing the number of public domain and licensed works that can be +freely distributed in machine readable form accessible by the widest +array of equipment including outdated equipment. Many small donations +($1 to $5,000) are particularly important to maintaining tax exempt +status with the IRS. + +The Foundation is committed to complying with the laws regulating +charities and charitable donations in all 50 states of the United +States. Compliance requirements are not uniform and it takes a +considerable effort, much paperwork and many fees to meet and keep up +with these requirements. We do not solicit donations in locations +where we have not received written confirmation of compliance. To +SEND DONATIONS or determine the status of compliance for any +particular state visit http://pglaf.org + +While we cannot and do not solicit contributions from states where we +have not met the solicitation requirements, we know of no prohibition +against accepting unsolicited donations from donors in such states who +approach us with offers to donate. + +International donations are gratefully accepted, but we cannot make +any statements concerning tax treatment of donations received from +outside the United States. U.S. laws alone swamp our small staff. + +Please check the Project Gutenberg Web pages for current donation +methods and addresses. Donations are accepted in a number of other +ways including checks, online payments and credit card donations. +To donate, please visit: http://pglaf.org/donate + + +Section 5. General Information About Project Gutenberg-tm electronic +works. + +Professor Michael S. Hart is the originator of the Project Gutenberg-tm +concept of a library of electronic works that could be freely shared +with anyone. For thirty years, he produced and distributed Project +Gutenberg-tm eBooks with only a loose network of volunteer support. + +Project Gutenberg-tm eBooks are often created from several printed +editions, all of which are confirmed as Public Domain in the U.S. +unless a copyright notice is included. Thus, we do not necessarily +keep eBooks in compliance with any particular paper edition. + +Each eBook is in a subdirectory of the same number as the eBook's +eBook number, often in several formats including plain vanilla ASCII, +compressed (zipped), HTML and others. + +Corrected EDITIONS of our eBooks replace the old file and take over +the old filename and etext number. The replaced older file is renamed. +VERSIONS based on separate sources are treated as new eBooks receiving +new filenames and etext numbers. + +Most people start at our Web site which has the main PG search facility: + +http://www.gutenberg.org + +This Web site includes information about Project Gutenberg-tm, +including how to make donations to the Project Gutenberg Literary +Archive Foundation, how to help produce our new eBooks, and how to +subscribe to our email newsletter to hear about new eBooks. + +EBooks posted prior to November 2003, with eBook numbers BELOW #10000, +are filed in directories based on their release date. If you want to +download any of these eBooks directly, rather than using the regular +search system you may utilize the following addresses and just +download by the etext year. + +http://www.ibiblio.org/gutenberg/etext06 + + (Or /etext 05, 04, 03, 02, 01, 00, 99, + 98, 97, 96, 95, 94, 93, 92, 92, 91 or 90) + +EBooks posted since November 2003, with etext numbers OVER #10000, are +filed in a different way. The year of a release date is no longer part +of the directory path. The path is based on the etext number (which is +identical to the filename). The path to the file is made up of single +digits corresponding to all but the last digit in the filename. For +example an eBook of filename 10234 would be found at: + +http://www.gutenberg.org/1/0/2/3/10234 + +or filename 24689 would be found at: +http://www.gutenberg.org/2/4/6/8/24689 + +An alternative method of locating eBooks: +http://www.gutenberg.org/GUTINDEX.ALL + +*** END: FULL LICENSE *** diff --git a/testing/tests/client/test_app.py b/testing/tests/client/test_app.py new file mode 100644 index 00000000..fef2f371 --- /dev/null +++ b/testing/tests/client/test_app.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# test_soledad_app.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test ObjectStore and Couch backend bits. +""" +from testscenarios import TestWithScenarios + +from test_soledad.util import BaseSoledadTest +from test_soledad.util import make_soledad_document_for_test +from test_soledad.util import make_token_soledad_app +from test_soledad.util import make_token_http_database_for_test +from test_soledad.util import copy_token_http_database_for_test +from test_soledad.u1db_tests import test_backends + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_backends`. +# ----------------------------------------------------------------------------- + +class SoledadTests( + TestWithScenarios, test_backends.AllDatabaseTests, BaseSoledadTest): + + scenarios = [ + ('token_http', { + 'make_database_for_test': make_token_http_database_for_test, + 'copy_database_for_test': copy_token_http_database_for_test, + 'make_document_for_test': make_soledad_document_for_test, + 'make_app_with_state': make_token_soledad_app, + }) + ] diff --git a/testing/tests/client/test_async.py b/testing/tests/client/test_async.py new file mode 100644 index 00000000..2ff70864 --- /dev/null +++ b/testing/tests/client/test_async.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- +# test_async.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import os +import hashlib + +from twisted.internet import defer + +from test_soledad.util import BaseSoledadTest +from leap.soledad.client import adbapi +from leap.soledad.client.sqlcipher import SQLCipherOptions + + +class ASyncSQLCipherRetryTestCase(BaseSoledadTest): + + """ + Test asynchronous SQLCipher operation. + """ + + NUM_DOCS = 5000 + + def _get_dbpool(self): + tmpdb = os.path.join(self.tempdir, "test.soledad") + opts = SQLCipherOptions(tmpdb, "secret", create=True) + return adbapi.getConnectionPool(opts) + + def _get_sample(self): + if not getattr(self, "_sample", None): + dirname = os.path.dirname(os.path.realpath(__file__)) + sample_file = os.path.join(dirname, "hacker_crackdown.txt") + with open(sample_file) as f: + self._sample = f.readlines() + return self._sample + + def test_concurrent_puts_fail_with_few_retries_and_small_timeout(self): + """ + Test if concurrent updates to the database with small timeout and + small number of retries fail with "database is locked" error. + + Many concurrent write attempts to the same sqlcipher database may fail + when the timeout is small and there are no retries. This test will + pass if any of the attempts to write the database fail. + + This test is much dependent on the environment and its result intends + to contrast with the test for the workaround for the "database is + locked" problem, which is addressed by the "test_concurrent_puts" test + below. + + If this test ever fails, it means that either (1) the platform where + you are running is it very powerful and you should try with an even + lower timeout value, or (2) the bug has been solved by a better + implementation of the underlying database pool, and thus this test + should be removed from the test suite. + """ + + old_timeout = adbapi.SQLCIPHER_CONNECTION_TIMEOUT + old_max_retries = adbapi.SQLCIPHER_MAX_RETRIES + + adbapi.SQLCIPHER_CONNECTION_TIMEOUT = 1 + adbapi.SQLCIPHER_MAX_RETRIES = 1 + + dbpool = self._get_dbpool() + + def _create_doc(doc): + return dbpool.runU1DBQuery("create_doc", doc) + + def _insert_docs(): + deferreds = [] + for i in range(self.NUM_DOCS): + payload = self._get_sample()[i] + chash = hashlib.sha256(payload).hexdigest() + doc = {"number": i, "payload": payload, 'chash': chash} + d = _create_doc(doc) + deferreds.append(d) + return defer.gatherResults(deferreds, consumeErrors=True) + + def _errback(e): + if e.value[0].getErrorMessage() == "database is locked": + adbapi.SQLCIPHER_CONNECTION_TIMEOUT = old_timeout + adbapi.SQLCIPHER_MAX_RETRIES = old_max_retries + return defer.succeed("") + raise Exception + + d = _insert_docs() + d.addCallback(lambda _: dbpool.runU1DBQuery("get_all_docs")) + d.addErrback(_errback) + return d + + def test_concurrent_puts(self): + """ + Test that many concurrent puts succeed. + + Currently, there's a known problem with the concurrent database pool + which is that many concurrent attempts to write to the database may + fail when the lock timeout is small and when there are no (or few) + retries. We currently workaround this problem by increasing the + timeout and the number of retries. + + Should this test ever fail, it probably means that the timeout and/or + number of retries should be increased for the platform you're running + the test. If the underlying database pool is ever fixed, then the test + above will fail and we should remove this comment from here. + """ + + dbpool = self._get_dbpool() + + def _create_doc(doc): + return dbpool.runU1DBQuery("create_doc", doc) + + def _insert_docs(): + deferreds = [] + for i in range(self.NUM_DOCS): + payload = self._get_sample()[i] + chash = hashlib.sha256(payload).hexdigest() + doc = {"number": i, "payload": payload, 'chash': chash} + d = _create_doc(doc) + deferreds.append(d) + return defer.gatherResults(deferreds, consumeErrors=True) + + def _count_docs(results): + _, docs = results + if self.NUM_DOCS == len(docs): + return defer.succeed("") + raise Exception + + d = _insert_docs() + d.addCallback(lambda _: dbpool.runU1DBQuery("get_all_docs")) + d.addCallback(_count_docs) + return d diff --git a/testing/tests/client/test_aux_methods.py b/testing/tests/client/test_aux_methods.py new file mode 100644 index 00000000..c25ff8ca --- /dev/null +++ b/testing/tests/client/test_aux_methods.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- +# test_soledad.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for general Soledad functionality. +""" +import os + +from twisted.internet import defer + +from leap.soledad.common.errors import DatabaseAccessError +from leap.soledad.client import Soledad +from leap.soledad.client.adbapi import U1DBConnectionPool +from leap.soledad.client.secrets import PassphraseTooShort + +from test_soledad.util import BaseSoledadTest + + +class AuxMethodsTestCase(BaseSoledadTest): + + def test__init_dirs(self): + sol = self._soledad_instance(prefix='_init_dirs') + local_db_dir = os.path.dirname(sol.local_db_path) + secrets_path = os.path.dirname(sol.secrets.secrets_path) + self.assertTrue(os.path.isdir(local_db_dir)) + self.assertTrue(os.path.isdir(secrets_path)) + + def _close_soledad(results): + sol.close() + + d = sol.create_doc({}) + d.addCallback(_close_soledad) + return d + + def test__init_u1db_sqlcipher_backend(self): + sol = self._soledad_instance(prefix='_init_db') + self.assertIsInstance(sol._dbpool, U1DBConnectionPool) + self.assertTrue(os.path.isfile(sol.local_db_path)) + sol.close() + + def test__init_config_with_defaults(self): + """ + Test if configuration defaults point to the correct place. + """ + + class SoledadMock(Soledad): + + def __init__(self): + pass + + # instantiate without initializing so we just test + # _init_config_with_defaults() + sol = SoledadMock() + sol._passphrase = u'' + sol._server_url = '' + sol._init_config_with_defaults() + # assert value of local_db_path + self.assertEquals( + os.path.join(sol.default_prefix, 'soledad.u1db'), + sol.local_db_path) + + def test__init_config_from_params(self): + """ + Test if configuration is correctly read from file. + """ + sol = self._soledad_instance( + 'leap@leap.se', + passphrase=u'123', + secrets_path='value_3', + local_db_path='value_2', + server_url='value_1', + cert_file=None) + self.assertEqual( + os.path.join(self.tempdir, 'value_3'), + sol.secrets.secrets_path) + self.assertEqual( + os.path.join(self.tempdir, 'value_2'), + sol.local_db_path) + self.assertEqual('value_1', sol._server_url) + sol.close() + + @defer.inlineCallbacks + def test_change_passphrase(self): + """ + Test if passphrase can be changed. + """ + prefix = '_change_passphrase' + sol = self._soledad_instance( + 'leap@leap.se', + passphrase=u'123', + prefix=prefix, + ) + + doc1 = yield sol.create_doc({'simple': 'doc'}) + sol.change_passphrase(u'654321') + sol.close() + + with self.assertRaises(DatabaseAccessError): + self._soledad_instance( + 'leap@leap.se', + passphrase=u'123', + prefix=prefix) + + sol2 = self._soledad_instance( + 'leap@leap.se', + passphrase=u'654321', + prefix=prefix) + doc2 = yield sol2.get_doc(doc1.doc_id) + + self.assertEqual(doc1, doc2) + + sol2.close() + + def test_change_passphrase_with_short_passphrase_raises(self): + """ + Test if attempt to change passphrase passing a short passphrase + raises. + """ + sol = self._soledad_instance( + 'leap@leap.se', + passphrase=u'123') + # check that soledad complains about new passphrase length + self.assertRaises( + PassphraseTooShort, + sol.change_passphrase, u'54321') + sol.close() + + def test_get_passphrase(self): + """ + Assert passphrase getter works fine. + """ + sol = self._soledad_instance() + self.assertEqual('123', sol._passphrase) + sol.close() diff --git a/testing/tests/client/test_crypto.py b/testing/tests/client/test_crypto.py new file mode 100644 index 00000000..77252b46 --- /dev/null +++ b/testing/tests/client/test_crypto.py @@ -0,0 +1,226 @@ +# -*- coding: utf-8 -*- +# test_crypto.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for cryptographic related stuff. +""" +import os +import hashlib +import binascii + +from leap.soledad.client import crypto +from leap.soledad.common.document import SoledadDocument +from test_soledad.util import BaseSoledadTest +from leap.soledad.common.crypto import WrongMacError +from leap.soledad.common.crypto import UnknownMacMethodError +from leap.soledad.common.crypto import ENC_JSON_KEY +from leap.soledad.common.crypto import ENC_SCHEME_KEY +from leap.soledad.common.crypto import MAC_KEY +from leap.soledad.common.crypto import MAC_METHOD_KEY + + +class EncryptedSyncTestCase(BaseSoledadTest): + + """ + Tests that guarantee that data will always be encrypted when syncing. + """ + + def test_encrypt_decrypt_json(self): + """ + Test encrypting and decrypting documents. + """ + simpledoc = {'key': 'val'} + doc1 = SoledadDocument(doc_id='id') + doc1.content = simpledoc + + # encrypt doc + doc1.set_json(self._soledad._crypto.encrypt_doc(doc1)) + # assert content is different and includes keys + self.assertNotEqual( + simpledoc, doc1.content, + 'incorrect document encryption') + self.assertTrue(ENC_JSON_KEY in doc1.content) + self.assertTrue(ENC_SCHEME_KEY in doc1.content) + # decrypt doc + doc1.set_json(self._soledad._crypto.decrypt_doc(doc1)) + self.assertEqual( + simpledoc, doc1.content, 'incorrect document encryption') + + +class RecoveryDocumentTestCase(BaseSoledadTest): + + def test_export_recovery_document_raw(self): + rd = self._soledad.secrets._export_recovery_document() + secret_id = rd[self._soledad.secrets.STORAGE_SECRETS_KEY].items()[0][0] + # assert exported secret is the same + secret = self._soledad.secrets._decrypt_storage_secret_version_1( + rd[self._soledad.secrets.STORAGE_SECRETS_KEY][secret_id]) + self.assertEqual(secret_id, self._soledad.secrets._secret_id) + self.assertEqual(secret, self._soledad.secrets._secrets[secret_id]) + # assert recovery document structure + encrypted_secret = rd[ + self._soledad.secrets.STORAGE_SECRETS_KEY][secret_id] + self.assertTrue(self._soledad.secrets.CIPHER_KEY in encrypted_secret) + self.assertTrue( + encrypted_secret[self._soledad.secrets.CIPHER_KEY] == 'aes256') + self.assertTrue(self._soledad.secrets.LENGTH_KEY in encrypted_secret) + self.assertTrue(self._soledad.secrets.SECRET_KEY in encrypted_secret) + + def test_import_recovery_document(self): + rd = self._soledad.secrets._export_recovery_document() + s = self._soledad_instance() + s.secrets._import_recovery_document(rd) + s.secrets.set_secret_id(self._soledad.secrets._secret_id) + self.assertEqual(self._soledad.storage_secret, + s.storage_secret, + 'Failed settinng secret for symmetric encryption.') + s.close() + + +class SoledadSecretsTestCase(BaseSoledadTest): + + def test_new_soledad_instance_generates_one_secret(self): + self.assertTrue( + self._soledad.storage_secret is not None, + "Expected secret to be something different than None") + number_of_secrets = len(self._soledad.secrets._secrets) + self.assertTrue( + number_of_secrets == 1, + "Expected exactly 1 secret, got %d instead." % number_of_secrets) + + def test_generated_secret_is_of_correct_type(self): + expected_type = str + self.assertIsInstance( + self._soledad.storage_secret, expected_type, + "Expected secret to be of type %s" % expected_type) + + def test_generated_secret_has_correct_lengt(self): + expected_length = self._soledad.secrets.GEN_SECRET_LENGTH + actual_length = len(self._soledad.storage_secret) + self.assertTrue( + expected_length == actual_length, + "Expected secret with length %d, got %d instead." + % (expected_length, actual_length)) + + def test_generated_secret_id_is_sha256_hash_of_secret(self): + generated = self._soledad.secrets.secret_id + expected = hashlib.sha256(self._soledad.storage_secret).hexdigest() + self.assertTrue( + generated == expected, + "Expeceted generated secret id to be sha256 hash, got something " + "else instead.") + + def test_generate_new_secret_generates_different_secret_id(self): + # generate new secret + secret_id_1 = self._soledad.secrets.secret_id + secret_id_2 = self._soledad.secrets._gen_secret() + self.assertTrue( + len(self._soledad.secrets._secrets) == 2, + "Expected exactly 2 secrets.") + self.assertTrue( + secret_id_1 != secret_id_2, + "Expected IDs of secrets to be distinct.") + self.assertTrue( + secret_id_1 in self._soledad.secrets._secrets, + "Expected to find ID of first secret in Soledad Secrets.") + self.assertTrue( + secret_id_2 in self._soledad.secrets._secrets, + "Expected to find ID of second secret in Soledad Secrets.") + + def test__has_secret(self): + self.assertTrue( + self._soledad._secrets._has_secret(), + "Should have a secret at this point") + + +class MacAuthTestCase(BaseSoledadTest): + + def test_decrypt_with_wrong_mac_raises(self): + """ + Trying to decrypt a document with wrong MAC should raise. + """ + simpledoc = {'key': 'val'} + doc = SoledadDocument(doc_id='id') + doc.content = simpledoc + # encrypt doc + doc.set_json(self._soledad._crypto.encrypt_doc(doc)) + self.assertTrue(MAC_KEY in doc.content) + self.assertTrue(MAC_METHOD_KEY in doc.content) + # mess with MAC + doc.content[MAC_KEY] = '1234567890ABCDEF' + # try to decrypt doc + self.assertRaises( + WrongMacError, + self._soledad._crypto.decrypt_doc, doc) + + def test_decrypt_with_unknown_mac_method_raises(self): + """ + Trying to decrypt a document with unknown MAC method should raise. + """ + simpledoc = {'key': 'val'} + doc = SoledadDocument(doc_id='id') + doc.content = simpledoc + # encrypt doc + doc.set_json(self._soledad._crypto.encrypt_doc(doc)) + self.assertTrue(MAC_KEY in doc.content) + self.assertTrue(MAC_METHOD_KEY in doc.content) + # mess with MAC method + doc.content[MAC_METHOD_KEY] = 'mymac' + # try to decrypt doc + self.assertRaises( + UnknownMacMethodError, + self._soledad._crypto.decrypt_doc, doc) + + +class SoledadCryptoAESTestCase(BaseSoledadTest): + + def test_encrypt_decrypt_sym(self): + # generate 256-bit key + key = os.urandom(32) + iv, cyphertext = crypto.encrypt_sym('data', key) + self.assertTrue(cyphertext is not None) + self.assertTrue(cyphertext != '') + self.assertTrue(cyphertext != 'data') + plaintext = crypto.decrypt_sym(cyphertext, key, iv) + self.assertEqual('data', plaintext) + + def test_decrypt_with_wrong_iv_fails(self): + key = os.urandom(32) + iv, cyphertext = crypto.encrypt_sym('data', key) + self.assertTrue(cyphertext is not None) + self.assertTrue(cyphertext != '') + self.assertTrue(cyphertext != 'data') + # get a different iv by changing the first byte + rawiv = binascii.a2b_base64(iv) + wrongiv = rawiv + while wrongiv == rawiv: + wrongiv = os.urandom(1) + rawiv[1:] + plaintext = crypto.decrypt_sym( + cyphertext, key, iv=binascii.b2a_base64(wrongiv)) + self.assertNotEqual('data', plaintext) + + def test_decrypt_with_wrong_key_fails(self): + key = os.urandom(32) + iv, cyphertext = crypto.encrypt_sym('data', key) + self.assertTrue(cyphertext is not None) + self.assertTrue(cyphertext != '') + self.assertTrue(cyphertext != 'data') + wrongkey = os.urandom(32) # 256-bits key + # ensure keys are different in case we are extremely lucky + while wrongkey == key: + wrongkey = os.urandom(32) + plaintext = crypto.decrypt_sym(cyphertext, wrongkey, iv) + self.assertNotEqual('data', plaintext) diff --git a/testing/tests/client/test_doc.py b/testing/tests/client/test_doc.py new file mode 100644 index 00000000..e158d768 --- /dev/null +++ b/testing/tests/client/test_doc.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# test_soledad_doc.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test Leap backend bits: soledad docs +""" +from testscenarios import TestWithScenarios + +from test_soledad.u1db_tests import test_document +from test_soledad.util import BaseSoledadTest +from test_soledad.util import make_soledad_document_for_test + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_document`. +# ----------------------------------------------------------------------------- + +class TestSoledadDocument( + TestWithScenarios, + test_document.TestDocument, BaseSoledadTest): + + scenarios = ([( + 'leap', { + 'make_document_for_test': make_soledad_document_for_test})]) + + +class TestSoledadPyDocument( + TestWithScenarios, + test_document.TestPyDocument, BaseSoledadTest): + + scenarios = ([( + 'leap', { + 'make_document_for_test': make_soledad_document_for_test})]) diff --git a/testing/tests/client/test_http.py b/testing/tests/client/test_http.py new file mode 100644 index 00000000..47df4b4a --- /dev/null +++ b/testing/tests/client/test_http.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +# test_http.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test Leap backend bits: test http database +""" + +from twisted.trial import unittest + +from leap.soledad.client import auth +from leap.soledad.common.l2db.remote import http_database + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_http_database`. +# ----------------------------------------------------------------------------- + +class _HTTPDatabase(http_database.HTTPDatabase, auth.TokenBasedAuth): + + """ + Wraps our token auth implementation. + """ + + def set_token_credentials(self, uuid, token): + auth.TokenBasedAuth.set_token_credentials(self, uuid, token) + + def _sign_request(self, method, url_query, params): + return auth.TokenBasedAuth._sign_request( + self, method, url_query, params) + + +class TestHTTPDatabaseWithCreds(unittest.TestCase): + + def test_get_sync_target_inherits_token_credentials(self): + # this test was from TestDatabaseSimpleOperations but we put it here + # for convenience. + self.db = _HTTPDatabase('dbase') + self.db.set_token_credentials('user-uuid', 'auth-token') + st = self.db.get_sync_target() + self.assertEqual(self.db._creds, st._creds) + + def test_ctr_with_creds(self): + db1 = _HTTPDatabase('http://dbs/db', creds={'token': { + 'uuid': 'user-uuid', + 'token': 'auth-token', + }}) + self.assertIn('token', db1._creds) diff --git a/testing/tests/client/test_http_client.py b/testing/tests/client/test_http_client.py new file mode 100644 index 00000000..a107930a --- /dev/null +++ b/testing/tests/client/test_http_client.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- +# test_http_client.py +# Copyright (C) 2013-2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test Leap backend bits: sync target +""" +import json + +from testscenarios import TestWithScenarios + +from leap.soledad.client import auth +from leap.soledad.common.l2db.remote import http_client +from test_soledad.u1db_tests import test_http_client +from leap.soledad.server.auth import SoledadTokenAuthMiddleware + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_http_client`. +# ----------------------------------------------------------------------------- + +class TestSoledadClientBase( + TestWithScenarios, + test_http_client.TestHTTPClientBase): + + """ + This class should be used to test Token auth. + """ + + def getClient(self, **kwds): + cli = self.getClientWithToken(**kwds) + if 'creds' not in kwds: + cli.set_token_credentials('user-uuid', 'auth-token') + return cli + + def getClientWithToken(self, **kwds): + self.startServer() + + class _HTTPClientWithToken( + http_client.HTTPClientBase, auth.TokenBasedAuth): + + def set_token_credentials(self, uuid, token): + auth.TokenBasedAuth.set_token_credentials(self, uuid, token) + + def _sign_request(self, method, url_query, params): + return auth.TokenBasedAuth._sign_request( + self, method, url_query, params) + + return _HTTPClientWithToken(self.getURL('dbase'), **kwds) + + def app(self, environ, start_response): + res = test_http_client.TestHTTPClientBase.app( + self, environ, start_response) + if res is not None: + return res + # mime solead application here. + if '/token' in environ['PATH_INFO']: + auth = environ.get(SoledadTokenAuthMiddleware.HTTP_AUTH_KEY) + if not auth: + start_response("401 Unauthorized", + [('Content-Type', 'application/json')]) + return [ + json.dumps( + {"error": "unauthorized", + "message": "no token found in environment"}) + ] + scheme, encoded = auth.split(None, 1) + if scheme.lower() != 'token': + start_response("401 Unauthorized", + [('Content-Type', 'application/json')]) + return [json.dumps({"error": "unauthorized", + "message": "unknown scheme: %s" % scheme})] + uuid, token = encoded.decode('base64').split(':', 1) + if uuid != 'user-uuid' and token != 'auth-token': + return Exception("Incorrect address or token.") + start_response("200 OK", [('Content-Type', 'application/json')]) + return [json.dumps([environ['PATH_INFO'], uuid, token])] + + def test_token(self): + """ + Test if token is sent correctly. + """ + cli = self.getClientWithToken() + cli.set_token_credentials('user-uuid', 'auth-token') + res, headers = cli._request('GET', ['doc', 'token']) + self.assertEqual( + ['/dbase/doc/token', 'user-uuid', 'auth-token'], json.loads(res)) + + def test_token_ctr_creds(self): + cli = self.getClientWithToken(creds={'token': { + 'uuid': 'user-uuid', + 'token': 'auth-token', + }}) + res, headers = cli._request('GET', ['doc', 'token']) + self.assertEqual( + ['/dbase/doc/token', 'user-uuid', 'auth-token'], json.loads(res)) diff --git a/testing/tests/client/test_https.py b/testing/tests/client/test_https.py new file mode 100644 index 00000000..caac16da --- /dev/null +++ b/testing/tests/client/test_https.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- +# test_sync_target.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test Leap backend bits: https +""" +from unittest import skip + +from testscenarios import TestWithScenarios + +from leap.soledad import client + +from leap.soledad.common.l2db.remote import http_client +from test_soledad.u1db_tests import test_backends +from test_soledad.u1db_tests import test_https +from test_soledad.util import ( + BaseSoledadTest, + make_soledad_document_for_test, + make_soledad_app, + make_token_soledad_app, +) + + +LEAP_SCENARIOS = [ + ('http', { + 'make_database_for_test': test_backends.make_http_database_for_test, + 'copy_database_for_test': test_backends.copy_http_database_for_test, + 'make_document_for_test': make_soledad_document_for_test, + 'make_app_with_state': make_soledad_app}), +] + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_https`. +# ----------------------------------------------------------------------------- + +def token_leap_https_sync_target(test, host, path, cert_file=None): + _, port = test.server.server_address + # source_replica_uid = test._soledad._dbpool.replica_uid + creds = {'token': {'uuid': 'user-uuid', 'token': 'auth-token'}} + if not cert_file: + cert_file = test.cacert_pem + st = client.http_target.SoledadHTTPSyncTarget( + 'https://%s:%d/%s' % (host, port, path), + source_replica_uid='other-id', + creds=creds, + crypto=test._soledad._crypto, + cert_file=cert_file) + return st + + +@skip("Skiping tests imported from U1DB.") +class TestSoledadHTTPSyncTargetHttpsSupport( + TestWithScenarios, + # test_https.TestHttpSyncTargetHttpsSupport, + BaseSoledadTest): + + scenarios = [ + ('token_soledad_https', + { + # 'server_def': test_https.https_server_def, + 'make_app_with_state': make_token_soledad_app, + 'make_document_for_test': make_soledad_document_for_test, + 'sync_target': token_leap_https_sync_target}), + ] + + def setUp(self): + # the parent constructor undoes our SSL monkey patch to ensure tests + # run smoothly with standard u1db. + test_https.TestHttpSyncTargetHttpsSupport.setUp(self) + # so here monkey patch again to test our functionality. + api = client.api + http_client._VerifiedHTTPSConnection = api.VerifiedHTTPSConnection + client.api.SOLEDAD_CERT = http_client.CA_CERTS + + def test_cannot_verify_cert(self): + self.startServer() + # don't print expected traceback server-side + self.server.handle_error = lambda req, cli_addr: None + self.request_state._create_database('test') + remote_target = self.getSyncTarget( + 'localhost', 'test', cert_file=http_client.CA_CERTS) + d = remote_target.record_sync_info('other-id', 2, 'T-id') + + def _assert_raises(result): + from twisted.python.failure import Failure + if isinstance(result, Failure): + from OpenSSL.SSL import Error + error = result.value.message[0].value + if isinstance(error, Error): + msg = error.message[0][2] + self.assertEqual("certificate verify failed", msg) + return + self.fail("certificate verification should have failed.") + + d.addCallbacks(_assert_raises, _assert_raises) + return d + + def test_working(self): + """ + Test that SSL connections work well. + + This test was adapted to patch Soledad's HTTPS connection custom class + with the intended CA certificates. + """ + self.startServer() + db = self.request_state._create_database('test') + remote_target = self.getSyncTarget('localhost', 'test') + d = remote_target.record_sync_info('other-id', 2, 'T-id') + d.addCallback(lambda _: + self.assertEqual( + (2, 'T-id'), + db._get_replica_gen_and_trans_id('other-id') + )) + d.addCallback(lambda _: remote_target.close()) + return d + + def test_host_mismatch(self): + """ + This test is disabled because soledad's twisted-based http agent uses + pyOpenSSL, which will complain if we try to use an IP to connect to + the remote host (see the original test in u1db_tests/test_https.py). + """ + pass diff --git a/testing/tests/client/test_shared_db.py b/testing/tests/client/test_shared_db.py new file mode 100644 index 00000000..aac766c2 --- /dev/null +++ b/testing/tests/client/test_shared_db.py @@ -0,0 +1,50 @@ +from leap.soledad.common.document import SoledadDocument +from leap.soledad.client.shared_db import SoledadSharedDatabase + +from test_soledad.util import BaseSoledadTest +from test_soledad.util import ADDRESS + + +class SoledadSharedDBTestCase(BaseSoledadTest): + + """ + These tests ensure the functionalities of the shared recovery database. + """ + + def setUp(self): + BaseSoledadTest.setUp(self) + self._shared_db = SoledadSharedDatabase( + 'https://provider/', ADDRESS, document_factory=SoledadDocument, + creds=None) + + def tearDown(self): + BaseSoledadTest.tearDown(self) + + def test__get_secrets_from_shared_db(self): + """ + Ensure the shared db is queried with the correct doc_id. + """ + doc_id = self._soledad.secrets._shared_db_doc_id() + self._soledad.secrets._get_secrets_from_shared_db() + self.assertTrue( + self._soledad.shared_db.get_doc.assert_called_with( + doc_id) is None, + 'Wrong doc_id when fetching recovery document.') + + def test__put_secrets_in_shared_db(self): + """ + Ensure recovery document is put into shared recover db. + """ + doc_id = self._soledad.secrets._shared_db_doc_id() + self._soledad.secrets._put_secrets_in_shared_db() + self.assertTrue( + self._soledad.shared_db.get_doc.assert_called_with( + doc_id) is None, + 'Wrong doc_id when fetching recovery document.') + self.assertTrue( + self._soledad.shared_db.put_doc.assert_called_with( + self._doc_put) is None, + 'Wrong document when putting recovery document.') + self.assertTrue( + self._doc_put.doc_id == doc_id, + 'Wrong doc_id when putting recovery document.') diff --git a/testing/tests/client/test_signals.py b/testing/tests/client/test_signals.py new file mode 100644 index 00000000..4e9ebfd0 --- /dev/null +++ b/testing/tests/client/test_signals.py @@ -0,0 +1,165 @@ +from mock import Mock +from twisted.internet import defer + +from leap import soledad +from leap.common.events import catalog +from leap.soledad.common.document import SoledadDocument + +from test_soledad.util import ADDRESS +from test_soledad.util import BaseSoledadTest + + +class SoledadSignalingTestCase(BaseSoledadTest): + + """ + These tests ensure signals are correctly emmited by Soledad. + """ + + EVENTS_SERVER_PORT = 8090 + + def setUp(self): + # mock signaling + soledad.client.signal = Mock() + soledad.client.secrets.events.emit_async = Mock() + # run parent's setUp + BaseSoledadTest.setUp(self) + + def tearDown(self): + BaseSoledadTest.tearDown(self) + + def _pop_mock_call(self, mocked): + mocked.call_args_list.pop() + mocked.mock_calls.pop() + mocked.call_args = mocked.call_args_list[-1] + + def test_stage3_bootstrap_signals(self): + """ + Test that a fresh soledad emits all bootstrap signals. + + Signals are: + - downloading keys / done downloading keys. + - creating keys / done creating keys. + - downloading keys / done downloading keys. + - uploading keys / done uploading keys. + """ + soledad.client.secrets.events.emit_async.reset_mock() + # get a fresh instance so it emits all bootstrap signals + sol = self._soledad_instance( + secrets_path='alternative_stage3.json', + local_db_path='alternative_stage3.u1db') + # reverse call order so we can verify in the order the signals were + # expected + soledad.client.secrets.events.emit_async.mock_calls.reverse() + soledad.client.secrets.events.emit_async.call_args = \ + soledad.client.secrets.events.emit_async.call_args_list[0] + soledad.client.secrets.events.emit_async.call_args_list.reverse() + + user_data = {'userid': ADDRESS, 'uuid': ADDRESS} + + # downloading keys signals + soledad.client.secrets.events.emit_async.assert_called_with( + catalog.SOLEDAD_DOWNLOADING_KEYS, user_data + ) + self._pop_mock_call(soledad.client.secrets.events.emit_async) + soledad.client.secrets.events.emit_async.assert_called_with( + catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data + ) + # creating keys signals + self._pop_mock_call(soledad.client.secrets.events.emit_async) + soledad.client.secrets.events.emit_async.assert_called_with( + catalog.SOLEDAD_CREATING_KEYS, user_data + ) + self._pop_mock_call(soledad.client.secrets.events.emit_async) + soledad.client.secrets.events.emit_async.assert_called_with( + catalog.SOLEDAD_DONE_CREATING_KEYS, user_data + ) + # downloading once more (inside _put_keys_in_shared_db) + self._pop_mock_call(soledad.client.secrets.events.emit_async) + soledad.client.secrets.events.emit_async.assert_called_with( + catalog.SOLEDAD_DOWNLOADING_KEYS, user_data + ) + self._pop_mock_call(soledad.client.secrets.events.emit_async) + soledad.client.secrets.events.emit_async.assert_called_with( + catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, user_data + ) + # uploading keys signals + self._pop_mock_call(soledad.client.secrets.events.emit_async) + soledad.client.secrets.events.emit_async.assert_called_with( + catalog.SOLEDAD_UPLOADING_KEYS, user_data + ) + self._pop_mock_call(soledad.client.secrets.events.emit_async) + soledad.client.secrets.events.emit_async.assert_called_with( + catalog.SOLEDAD_DONE_UPLOADING_KEYS, user_data + ) + sol.close() + + def test_stage2_bootstrap_signals(self): + """ + Test that if there are keys in server, soledad will download them and + emit corresponding signals. + """ + # get existing instance so we have access to keys + sol = self._soledad_instance() + # create a document with secrets + doc = SoledadDocument(doc_id=sol.secrets._shared_db_doc_id()) + doc.content = sol.secrets._export_recovery_document() + sol.close() + # reset mock + soledad.client.secrets.events.emit_async.reset_mock() + # get a fresh instance so it emits all bootstrap signals + shared_db = self.get_default_shared_mock(get_doc_return_value=doc) + sol = self._soledad_instance( + secrets_path='alternative_stage2.json', + local_db_path='alternative_stage2.u1db', + shared_db_class=shared_db) + # reverse call order so we can verify in the order the signals were + # expected + soledad.client.secrets.events.emit_async.mock_calls.reverse() + soledad.client.secrets.events.emit_async.call_args = \ + soledad.client.secrets.events.emit_async.call_args_list[0] + soledad.client.secrets.events.emit_async.call_args_list.reverse() + # assert download keys signals + soledad.client.secrets.events.emit_async.assert_called_with( + catalog.SOLEDAD_DOWNLOADING_KEYS, + {'userid': ADDRESS, 'uuid': ADDRESS} + ) + self._pop_mock_call(soledad.client.secrets.events.emit_async) + soledad.client.secrets.events.emit_async.assert_called_with( + catalog.SOLEDAD_DONE_DOWNLOADING_KEYS, + {'userid': ADDRESS, 'uuid': ADDRESS}, + ) + sol.close() + + def test_stage1_bootstrap_signals(self): + """ + Test that if soledad already has a local secret, it emits no signals. + """ + soledad.client.signal.reset_mock() + # get an existent instance so it emits only some of bootstrap signals + sol = self._soledad_instance() + self.assertEqual([], soledad.client.signal.mock_calls) + sol.close() + + @defer.inlineCallbacks + def test_sync_signals(self): + """ + Test Soledad emits SOLEDAD_CREATING_KEYS signal. + """ + # get a fresh instance so it emits all bootstrap signals + sol = self._soledad_instance() + soledad.client.signal.reset_mock() + + # mock the actual db sync so soledad does not try to connect to the + # server + d = defer.Deferred() + d.callback(None) + sol._dbsyncer.sync = Mock(return_value=d) + + yield sol.sync() + + # assert the signal has been emitted + soledad.client.events.emit_async.assert_called_with( + catalog.SOLEDAD_DONE_DATA_SYNC, + {'userid': ADDRESS, 'uuid': ADDRESS}, + ) + sol.close() diff --git a/testing/tests/client/test_soledad_doc.py b/testing/tests/client/test_soledad_doc.py new file mode 100644 index 00000000..e158d768 --- /dev/null +++ b/testing/tests/client/test_soledad_doc.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# test_soledad_doc.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test Leap backend bits: soledad docs +""" +from testscenarios import TestWithScenarios + +from test_soledad.u1db_tests import test_document +from test_soledad.util import BaseSoledadTest +from test_soledad.util import make_soledad_document_for_test + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_document`. +# ----------------------------------------------------------------------------- + +class TestSoledadDocument( + TestWithScenarios, + test_document.TestDocument, BaseSoledadTest): + + scenarios = ([( + 'leap', { + 'make_document_for_test': make_soledad_document_for_test})]) + + +class TestSoledadPyDocument( + TestWithScenarios, + test_document.TestPyDocument, BaseSoledadTest): + + scenarios = ([( + 'leap', { + 'make_document_for_test': make_soledad_document_for_test})]) diff --git a/testing/tests/client/test_sqlcipher.py b/testing/tests/client/test_sqlcipher.py new file mode 100644 index 00000000..11472d46 --- /dev/null +++ b/testing/tests/client/test_sqlcipher.py @@ -0,0 +1,705 @@ +# -*- coding: utf-8 -*- +# test_sqlcipher.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test sqlcipher backend internals. +""" +import os +import time +import threading +import tempfile +import shutil + +from pysqlcipher import dbapi2 +from testscenarios import TestWithScenarios + +# l2db stuff. +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db import query_parser +from leap.soledad.common.l2db.backends.sqlite_backend \ + import SQLitePartialExpandDatabase + +# soledad stuff. +from leap.soledad.common import soledad_assert +from leap.soledad.common.document import SoledadDocument +from leap.soledad.client.sqlcipher import SQLCipherDatabase +from leap.soledad.client.sqlcipher import SQLCipherOptions +from leap.soledad.client.sqlcipher import DatabaseIsNotEncrypted + +# u1db tests stuff. +from test_soledad import u1db_tests as tests +from test_soledad.u1db_tests import test_backends +from test_soledad.u1db_tests import test_open +from test_soledad.util import SQLCIPHER_SCENARIOS +from test_soledad.util import PASSWORD +from test_soledad.util import BaseSoledadTest + + +def sqlcipher_open(path, passphrase, create=True, document_factory=None): + return SQLCipherDatabase( + SQLCipherOptions(path, passphrase, create=create)) + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_common_backend`. +# ----------------------------------------------------------------------------- + +class TestSQLCipherBackendImpl(tests.TestCase): + + def test__allocate_doc_id(self): + db = sqlcipher_open(':memory:', PASSWORD) + doc_id1 = db._allocate_doc_id() + self.assertTrue(doc_id1.startswith('D-')) + self.assertEqual(34, len(doc_id1)) + int(doc_id1[len('D-'):], 16) + self.assertNotEqual(doc_id1, db._allocate_doc_id()) + db.close() + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_backends`. +# ----------------------------------------------------------------------------- + +class SQLCipherTests(TestWithScenarios, test_backends.AllDatabaseTests): + scenarios = SQLCIPHER_SCENARIOS + + +class SQLCipherDatabaseTests(TestWithScenarios, + test_backends.LocalDatabaseTests): + scenarios = SQLCIPHER_SCENARIOS + + +class SQLCipherValidateGenNTransIdTests( + TestWithScenarios, + test_backends.LocalDatabaseValidateGenNTransIdTests): + scenarios = SQLCIPHER_SCENARIOS + + +class SQLCipherValidateSourceGenTests( + TestWithScenarios, + test_backends.LocalDatabaseValidateSourceGenTests): + scenarios = SQLCIPHER_SCENARIOS + + +class SQLCipherWithConflictsTests( + TestWithScenarios, + test_backends.LocalDatabaseWithConflictsTests): + scenarios = SQLCIPHER_SCENARIOS + + +class SQLCipherIndexTests( + TestWithScenarios, test_backends.DatabaseIndexTests): + scenarios = SQLCIPHER_SCENARIOS + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_sqlite_backend`. +# ----------------------------------------------------------------------------- + +class TestSQLCipherDatabase(tests.TestCase): + """ + Tests from u1db.tests.test_sqlite_backend.TestSQLiteDatabase. + """ + + def test_atomic_initialize(self): + # This test was modified to ensure that db2.close() is called within + # the thread that created the database. + tmpdir = self.createTempDir() + dbname = os.path.join(tmpdir, 'atomic.db') + + t2 = None # will be a thread + + class SQLCipherDatabaseTesting(SQLCipherDatabase): + _index_storage_value = "testing" + + def __init__(self, dbname, ntry): + self._try = ntry + self._is_initialized_invocations = 0 + SQLCipherDatabase.__init__( + self, + SQLCipherOptions(dbname, PASSWORD)) + + def _is_initialized(self, c): + res = \ + SQLCipherDatabase._is_initialized(self, c) + if self._try == 1: + self._is_initialized_invocations += 1 + if self._is_initialized_invocations == 2: + t2.start() + # hard to do better and have a generic test + time.sleep(0.05) + return res + + class SecondTry(threading.Thread): + + outcome2 = [] + + def run(self): + try: + db2 = SQLCipherDatabaseTesting(dbname, 2) + except Exception, e: + SecondTry.outcome2.append(e) + else: + SecondTry.outcome2.append(db2) + + t2 = SecondTry() + db1 = SQLCipherDatabaseTesting(dbname, 1) + t2.join() + + self.assertIsInstance(SecondTry.outcome2[0], SQLCipherDatabaseTesting) + self.assertTrue(db1._is_initialized(db1._get_sqlite_handle().cursor())) + db1.close() + + +class TestSQLCipherPartialExpandDatabase(tests.TestCase): + """ + Tests from u1db.tests.test_sqlite_backend.TestSQLitePartialExpandDatabase. + """ + + # The following tests had to be cloned from u1db because they all + # instantiate the backend directly, so we need to change that in order to + # our backend be instantiated in place. + + def setUp(self): + self.db = sqlcipher_open(':memory:', PASSWORD) + + def tearDown(self): + self.db.close() + + def test_default_replica_uid(self): + self.assertIsNot(None, self.db._replica_uid) + self.assertEqual(32, len(self.db._replica_uid)) + int(self.db._replica_uid, 16) + + def test__parse_index(self): + g = self.db._parse_index_definition('fieldname') + self.assertIsInstance(g, query_parser.ExtractField) + self.assertEqual(['fieldname'], g.field) + + def test__update_indexes(self): + g = self.db._parse_index_definition('fieldname') + c = self.db._get_sqlite_handle().cursor() + self.db._update_indexes('doc-id', {'fieldname': 'val'}, + [('fieldname', g)], c) + c.execute('SELECT doc_id, field_name, value FROM document_fields') + self.assertEqual([('doc-id', 'fieldname', 'val')], + c.fetchall()) + + def test_create_database(self): + raw_db = self.db._get_sqlite_handle() + self.assertNotEqual(None, raw_db) + + def test__set_replica_uid(self): + # Start from scratch, so that replica_uid isn't set. + self.assertIsNot(None, self.db._real_replica_uid) + self.assertIsNot(None, self.db._replica_uid) + self.db._set_replica_uid('foo') + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT value FROM u1db_config WHERE name='replica_uid'") + self.assertEqual(('foo',), c.fetchone()) + self.assertEqual('foo', self.db._real_replica_uid) + self.assertEqual('foo', self.db._replica_uid) + self.db._close_sqlite_handle() + self.assertEqual('foo', self.db._replica_uid) + + def test__open_database(self): + # SQLCipherDatabase has no _open_database() method, so we just pass + # (and test for the same funcionality on test_open_database_existing() + # below). + pass + + def test__open_database_with_factory(self): + # SQLCipherDatabase has no _open_database() method. + pass + + def test__open_database_non_existent(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/non-existent.sqlite' + self.assertRaises(errors.DatabaseDoesNotExist, + sqlcipher_open, + path, PASSWORD, create=False) + + def test__open_database_during_init(self): + # The purpose of this test is to ensure that _open_database() parallel + # db initialization behaviour is correct. As SQLCipherDatabase does + # not have an _open_database() method, we just do not implement this + # test. + pass + + def test__open_database_invalid(self): + # This test was modified to ensure that an empty database file will + # raise a DatabaseIsNotEncrypted exception instead of a + # dbapi2.OperationalError exception. + temp_dir = self.createTempDir(prefix='u1db-test-') + path1 = temp_dir + '/invalid1.db' + with open(path1, 'wb') as f: + f.write("") + self.assertRaises(DatabaseIsNotEncrypted, + sqlcipher_open, path1, + PASSWORD) + with open(path1, 'wb') as f: + f.write("invalid") + self.assertRaises(dbapi2.DatabaseError, + sqlcipher_open, path1, + PASSWORD) + + def test_open_database_existing(self): + # In the context of SQLCipherDatabase, where no _open_database() + # method exists and thus there's no call to _which_index_storage(), + # this test tests for the same functionality as + # test_open_database_create() below. So, we just pass. + pass + + def test_open_database_with_factory(self): + # SQLCipherDatabase's constructor has no factory parameter. + pass + + def test_open_database_create(self): + # SQLCipherDatabas has no open_database() method, so we just test for + # the actual database constructor effects. + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/new.sqlite' + db1 = sqlcipher_open(path, PASSWORD, create=True) + db2 = sqlcipher_open(path, PASSWORD, create=False) + self.assertIsInstance(db2, SQLCipherDatabase) + db1.close() + db2.close() + + def test_create_database_initializes_schema(self): + # This test had to be cloned because our implementation of SQLCipher + # backend is referenced with an index_storage_value that includes the + # word "encrypted". See u1db's sqlite_backend and our + # sqlcipher_backend for reference. + raw_db = self.db._get_sqlite_handle() + c = raw_db.cursor() + c.execute("SELECT * FROM u1db_config") + config = dict([(r[0], r[1]) for r in c.fetchall()]) + replica_uid = self.db._replica_uid + self.assertEqual({'sql_schema': '0', 'replica_uid': replica_uid, + 'index_storage': 'expand referenced encrypted'}, + config) + + def test_store_syncable(self): + doc = self.db.create_doc_from_json(tests.simple_doc) + # assert that docs are syncable by default + self.assertEqual(True, doc.syncable) + # assert that we can store syncable = False + doc.syncable = False + self.db.put_doc(doc) + self.assertEqual(False, self.db.get_doc(doc.doc_id).syncable) + # assert that we can store syncable = True + doc.syncable = True + self.db.put_doc(doc) + self.assertEqual(True, self.db.get_doc(doc.doc_id).syncable) + + def test__close_sqlite_handle(self): + raw_db = self.db._get_sqlite_handle() + self.db._close_sqlite_handle() + self.assertRaises(dbapi2.ProgrammingError, + raw_db.cursor) + + def test__get_generation(self): + self.assertEqual(0, self.db._get_generation()) + + def test__get_generation_info(self): + self.assertEqual((0, ''), self.db._get_generation_info()) + + def test_create_index(self): + self.db.create_index('test-idx', "key") + self.assertEqual([('test-idx', ["key"])], self.db.list_indexes()) + + def test_create_index_multiple_fields(self): + self.db.create_index('test-idx', "key", "key2") + self.assertEqual([('test-idx', ["key", "key2"])], + self.db.list_indexes()) + + def test__get_index_definition(self): + self.db.create_index('test-idx', "key", "key2") + # TODO: How would you test that an index is getting used for an SQL + # request? + self.assertEqual(["key", "key2"], + self.db._get_index_definition('test-idx')) + + def test_list_index_mixed(self): + # Make sure that we properly order the output + c = self.db._get_sqlite_handle().cursor() + # We intentionally insert the data in weird ordering, to make sure the + # query still gets it back correctly. + c.executemany("INSERT INTO index_definitions VALUES (?, ?, ?)", + [('idx-1', 0, 'key10'), + ('idx-2', 2, 'key22'), + ('idx-1', 1, 'key11'), + ('idx-2', 0, 'key20'), + ('idx-2', 1, 'key21')]) + self.assertEqual([('idx-1', ['key10', 'key11']), + ('idx-2', ['key20', 'key21', 'key22'])], + self.db.list_indexes()) + + def test_no_indexes_no_document_fields(self): + self.db.create_doc_from_json( + '{"key1": "val1", "key2": "val2"}') + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([], c.fetchall()) + + def test_create_extracts_fields(self): + doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') + doc2 = self.db.create_doc_from_json('{"key1": "valx", "key2": "valy"}') + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([], c.fetchall()) + self.db.create_index('test', 'key1', 'key2') + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual(sorted( + [(doc1.doc_id, "key1", "val1"), + (doc1.doc_id, "key2", "val2"), + (doc2.doc_id, "key1", "valx"), + (doc2.doc_id, "key2", "valy"), ]), sorted(c.fetchall())) + + def test_put_updates_fields(self): + self.db.create_index('test', 'key1', 'key2') + doc1 = self.db.create_doc_from_json( + '{"key1": "val1", "key2": "val2"}') + doc1.content = {"key1": "val1", "key2": "valy"} + self.db.put_doc(doc1) + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([(doc1.doc_id, "key1", "val1"), + (doc1.doc_id, "key2", "valy"), ], c.fetchall()) + + def test_put_updates_nested_fields(self): + self.db.create_index('test', 'key', 'sub.doc') + doc1 = self.db.create_doc_from_json(tests.nested_doc) + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([(doc1.doc_id, "key", "value"), + (doc1.doc_id, "sub.doc", "underneath"), ], + c.fetchall()) + + def test__ensure_schema_rollback(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/rollback.db' + + class SQLitePartialExpandDbTesting(SQLCipherDatabase): + + def _set_replica_uid_in_transaction(self, uid): + super(SQLitePartialExpandDbTesting, + self)._set_replica_uid_in_transaction(uid) + if fail: + raise Exception() + + db = SQLitePartialExpandDbTesting.__new__(SQLitePartialExpandDbTesting) + db._db_handle = dbapi2.connect(path) # db is there but not yet init-ed + fail = True + self.assertRaises(Exception, db._ensure_schema) + fail = False + db._initialize(db._db_handle.cursor()) + + def test_open_database_non_existent(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/non-existent.sqlite' + self.assertRaises(errors.DatabaseDoesNotExist, + sqlcipher_open, path, "123", + create=False) + + def test_delete_database_existent(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/new.sqlite' + db = sqlcipher_open(path, "123", create=True) + db.close() + SQLCipherDatabase.delete_database(path) + self.assertRaises(errors.DatabaseDoesNotExist, + sqlcipher_open, path, "123", + create=False) + + def test_delete_database_nonexistent(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/non-existent.sqlite' + self.assertRaises(errors.DatabaseDoesNotExist, + SQLCipherDatabase.delete_database, path) + + def test__get_indexed_fields(self): + self.db.create_index('idx1', 'a', 'b') + self.assertEqual(set(['a', 'b']), self.db._get_indexed_fields()) + self.db.create_index('idx2', 'b', 'c') + self.assertEqual(set(['a', 'b', 'c']), self.db._get_indexed_fields()) + + def test_indexed_fields_expanded(self): + self.db.create_index('idx1', 'key1') + doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') + self.assertEqual(set(['key1']), self.db._get_indexed_fields()) + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([(doc1.doc_id, 'key1', 'val1')], c.fetchall()) + + def test_create_index_updates_fields(self): + doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') + self.db.create_index('idx1', 'key1') + self.assertEqual(set(['key1']), self.db._get_indexed_fields()) + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([(doc1.doc_id, 'key1', 'val1')], c.fetchall()) + + def assertFormatQueryEquals(self, exp_statement, exp_args, definition, + values): + statement, args = self.db._format_query(definition, values) + self.assertEqual(exp_statement, statement) + self.assertEqual(exp_args, args) + + def test__format_query(self): + self.assertFormatQueryEquals( + "SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM " + "document d, document_fields d0 LEFT OUTER JOIN conflicts c ON " + "c.doc_id = d.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name " + "= ? AND d0.value = ? GROUP BY d.doc_id, d.doc_rev, d.content " + "ORDER BY d0.value;", ["key1", "a"], + ["key1"], ["a"]) + + def test__format_query2(self): + self.assertFormatQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value = ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value = ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value = ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' + 'd0.value, d1.value, d2.value;', + ["key1", "a", "key2", "b", "key3", "c"], + ["key1", "key2", "key3"], ["a", "b", "c"]) + + def test__format_query_wildcard(self): + self.assertFormatQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value = ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value GLOB ? AND d.doc_id = d2.doc_id AND d2.field_name = ? ' + 'AND d2.value NOT NULL GROUP BY d.doc_id, d.doc_rev, d.content ' + 'ORDER BY d0.value, d1.value, d2.value;', + ["key1", "a", "key2", "b*", "key3"], ["key1", "key2", "key3"], + ["a", "b*", "*"]) + + def assertFormatRangeQueryEquals(self, exp_statement, exp_args, definition, + start_value, end_value): + statement, args = self.db._format_range_query( + definition, start_value, end_value) + self.assertEqual(exp_statement, statement) + self.assertEqual(exp_args, args) + + def test__format_range_query(self): + self.assertFormatRangeQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value >= ? AND d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value <= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value <= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' + 'd0.value, d1.value, d2.value;', + ['key1', 'a', 'key2', 'b', 'key3', 'c', 'key1', 'p', 'key2', 'q', + 'key3', 'r'], + ["key1", "key2", "key3"], ["a", "b", "c"], ["p", "q", "r"]) + + def test__format_range_query_no_start(self): + self.assertFormatRangeQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value <= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value <= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' + 'd0.value, d1.value, d2.value;', + ['key1', 'a', 'key2', 'b', 'key3', 'c'], + ["key1", "key2", "key3"], None, ["a", "b", "c"]) + + def test__format_range_query_no_end(self): + self.assertFormatRangeQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value >= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' + 'd0.value, d1.value, d2.value;', + ['key1', 'a', 'key2', 'b', 'key3', 'c'], + ["key1", "key2", "key3"], ["a", "b", "c"], None) + + def test__format_range_query_wildcard(self): + self.assertFormatRangeQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value NOT NULL AND d.doc_id = d0.doc_id AND d0.field_name = ? ' + 'AND d0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? ' + 'AND (d1.value < ? OR d1.value GLOB ?) AND d.doc_id = d2.doc_id ' + 'AND d2.field_name = ? AND d2.value NOT NULL GROUP BY d.doc_id, ' + 'd.doc_rev, d.content ORDER BY d0.value, d1.value, d2.value;', + ['key1', 'a', 'key2', 'b', 'key3', 'key1', 'p', 'key2', 'q', 'q*', + 'key3'], + ["key1", "key2", "key3"], ["a", "b*", "*"], ["p", "q*", "*"]) + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_open`. +# ----------------------------------------------------------------------------- + + +class SQLCipherOpen(test_open.TestU1DBOpen): + + def test_open_no_create(self): + self.assertRaises(errors.DatabaseDoesNotExist, + sqlcipher_open, self.db_path, + PASSWORD, + create=False) + self.assertFalse(os.path.exists(self.db_path)) + + def test_open_create(self): + db = sqlcipher_open(self.db_path, PASSWORD, create=True) + self.addCleanup(db.close) + self.assertTrue(os.path.exists(self.db_path)) + self.assertIsInstance(db, SQLCipherDatabase) + + def test_open_with_factory(self): + db = sqlcipher_open(self.db_path, PASSWORD, create=True, + document_factory=SoledadDocument) + self.addCleanup(db.close) + doc = db.create_doc({}) + self.assertTrue(isinstance(doc, SoledadDocument)) + + def test_open_existing(self): + db = sqlcipher_open(self.db_path, PASSWORD) + self.addCleanup(db.close) + doc = db.create_doc_from_json(tests.simple_doc) + # Even though create=True, we shouldn't wipe the db + db2 = sqlcipher_open(self.db_path, PASSWORD, create=True) + self.addCleanup(db2.close) + doc2 = db2.get_doc(doc.doc_id) + self.assertEqual(doc, doc2) + + def test_open_existing_no_create(self): + db = sqlcipher_open(self.db_path, PASSWORD) + self.addCleanup(db.close) + db2 = sqlcipher_open(self.db_path, PASSWORD, create=False) + self.addCleanup(db2.close) + self.assertIsInstance(db2, SQLCipherDatabase) + + +# ----------------------------------------------------------------------------- +# Tests for actual encryption of the database +# ----------------------------------------------------------------------------- + +class SQLCipherEncryptionTests(BaseSoledadTest): + + """ + Tests to guarantee SQLCipher is indeed encrypting data when storing. + """ + + def _delete_dbfiles(self): + for dbfile in [self.DB_FILE]: + if os.path.exists(dbfile): + os.unlink(dbfile) + + def setUp(self): + # the following come from BaseLeapTest.setUpClass, because + # twisted.trial doesn't support such class methods for setting up + # test classes. + self.old_path = os.environ['PATH'] + self.old_home = os.environ['HOME'] + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + self.home = self.tempdir + bin_tdir = os.path.join( + self.tempdir, + 'bin') + os.environ["PATH"] = bin_tdir + os.environ["HOME"] = self.tempdir + # this is our own stuff + self.DB_FILE = os.path.join(self.tempdir, 'test.db') + self._delete_dbfiles() + + def tearDown(self): + self._delete_dbfiles() + # the following come from BaseLeapTest.tearDownClass, because + # twisted.trial doesn't support such class methods for tearing down + # test classes. + os.environ["PATH"] = self.old_path + os.environ["HOME"] = self.old_home + # safety check! please do not wipe my home... + # XXX needs to adapt to non-linuces + soledad_assert( + self.tempdir.startswith('/tmp/leap_tests-') or + self.tempdir.startswith('/var/folder'), + "beware! tried to remove a dir which does not " + "live in temporal folder!") + shutil.rmtree(self.tempdir) + + def test_try_to_open_encrypted_db_with_sqlite_backend(self): + """ + SQLite backend should not succeed to open SQLCipher databases. + """ + db = sqlcipher_open(self.DB_FILE, PASSWORD) + doc = db.create_doc_from_json(tests.simple_doc) + db.close() + try: + # trying to open an encrypted database with the regular u1db + # backend should raise a DatabaseError exception. + SQLitePartialExpandDatabase(self.DB_FILE, + document_factory=SoledadDocument) + raise DatabaseIsNotEncrypted() + except dbapi2.DatabaseError: + # at this point we know that the regular U1DB sqlcipher backend + # did not succeed on opening the database, so it was indeed + # encrypted. + db = sqlcipher_open(self.DB_FILE, PASSWORD) + doc = db.get_doc(doc.doc_id) + self.assertEqual(tests.simple_doc, doc.get_json(), + 'decrypted content mismatch') + db.close() + + def test_try_to_open_raw_db_with_sqlcipher_backend(self): + """ + SQLCipher backend should not succeed to open unencrypted databases. + """ + db = SQLitePartialExpandDatabase(self.DB_FILE, + document_factory=SoledadDocument) + db.create_doc_from_json(tests.simple_doc) + db.close() + try: + # trying to open the a non-encrypted database with sqlcipher + # backend should raise a DatabaseIsNotEncrypted exception. + db = sqlcipher_open(self.DB_FILE, PASSWORD) + db.close() + raise dbapi2.DatabaseError( + "SQLCipher backend should not be able to open non-encrypted " + "dbs.") + except DatabaseIsNotEncrypted: + pass diff --git a/testing/tests/client/test_sqlcipher_sync.py b/testing/tests/client/test_sqlcipher_sync.py new file mode 100644 index 00000000..3cbefc8b --- /dev/null +++ b/testing/tests/client/test_sqlcipher_sync.py @@ -0,0 +1,730 @@ +# -*- coding: utf-8 -*- +# test_sqlcipher.py +# Copyright (C) 2013-2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test sqlcipher backend sync. +""" +import os + +from uuid import uuid4 + +from testscenarios import TestWithScenarios + +from leap.soledad.common.l2db import sync +from leap.soledad.common.l2db import vectorclock +from leap.soledad.common.l2db import errors + +from leap.soledad.common.crypto import ENC_SCHEME_KEY +from leap.soledad.client.crypto import decrypt_doc_dict +from leap.soledad.client.http_target import SoledadHTTPSyncTarget + +from test_soledad import u1db_tests as tests +from test_soledad.util import SQLCIPHER_SCENARIOS +from test_soledad.util import make_soledad_app +from test_soledad.util import soledad_sync_target +from test_soledad.util import BaseSoledadTest + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_sync`. +# ----------------------------------------------------------------------------- + +def sync_via_synchronizer_and_soledad(test, db_source, db_target, + trace_hook=None, + trace_hook_shallow=None): + if trace_hook: + test.skipTest("full trace hook unsupported over http") + path = test._http_at[db_target] + target = SoledadHTTPSyncTarget.connect( + test.getURL(path), test._soledad._crypto) + target.set_token_credentials('user-uuid', 'auth-token') + if trace_hook_shallow: + target._set_trace_hook_shallow(trace_hook_shallow) + return sync.Synchronizer(db_source, target).sync() + + +def sync_via_synchronizer(test, db_source, db_target, + trace_hook=None, + trace_hook_shallow=None): + target = db_target.get_sync_target() + trace_hook = trace_hook or trace_hook_shallow + if trace_hook: + target._set_trace_hook(trace_hook) + return sync.Synchronizer(db_source, target).sync() + + +sync_scenarios = [] +for name, scenario in SQLCIPHER_SCENARIOS: + scenario['do_sync'] = sync_via_synchronizer + sync_scenarios.append((name, scenario)) + + +class SQLCipherDatabaseSyncTests( + TestWithScenarios, + tests.DatabaseBaseTests, + BaseSoledadTest): + + """ + Test for succesfull sync between SQLCipher and LeapBackend. + + Some of the tests in this class had to be adapted because the remote + backend always receive encrypted content, and so it can not rely on + document's content comparison to try to autoresolve conflicts. + """ + + scenarios = sync_scenarios + + def setUp(self): + self._use_tracking = {} + super(tests.DatabaseBaseTests, self).setUp() + + def create_database(self, replica_uid, sync_role=None): + if replica_uid == 'test' and sync_role is None: + # created up the chain by base class but unused + return None + db = self.create_database_for_role(replica_uid, sync_role) + if sync_role: + self._use_tracking[db] = (replica_uid, sync_role) + self.addCleanup(db.close) + return db + + def create_database_for_role(self, replica_uid, sync_role): + # hook point for reuse + return tests.DatabaseBaseTests.create_database(self, replica_uid) + + def sync(self, db_from, db_to, trace_hook=None, + trace_hook_shallow=None): + from_name, from_sync_role = self._use_tracking[db_from] + to_name, to_sync_role = self._use_tracking[db_to] + if from_sync_role not in ('source', 'both'): + raise Exception("%s marked for %s use but used as source" % + (from_name, from_sync_role)) + if to_sync_role not in ('target', 'both'): + raise Exception("%s marked for %s use but used as target" % + (to_name, to_sync_role)) + return self.do_sync(self, db_from, db_to, trace_hook, + trace_hook_shallow) + + def assertLastExchangeLog(self, db, expected): + log = getattr(db, '_last_exchange_log', None) + if log is None: + return + self.assertEqual(expected, log) + + def copy_database(self, db, sync_role=None): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES + # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST + # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS + # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND + # NINJA TO YOUR HOUSE. + db_copy = tests.DatabaseBaseTests.copy_database(self, db) + name, orig_sync_role = self._use_tracking[db] + self._use_tracking[db_copy] = (name + '(copy)', sync_role or + orig_sync_role) + return db_copy + + def test_sync_tracks_db_generation_of_other(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.assertEqual(0, self.sync(self.db1, self.db2)) + self.assertEqual( + (0, ''), self.db1._get_replica_gen_and_trans_id('test2')) + self.assertEqual( + (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) + self.assertLastExchangeLog(self.db2, + {'receive': + {'docs': [], 'last_known_gen': 0}, + 'return': + {'docs': [], 'last_gen': 0}}) + + def test_sync_autoresolves(self): + """ + Test for sync autoresolve remote. + + This test was adapted because the remote database receives encrypted + content and so it can't compare documents contents to autoresolve. + """ + # The remote database can't autoresolve conflicts based on magic + # content convergence, so we modify this test to leave the possibility + # of the remode document ending up in conflicted state. + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc') + rev1 = doc1.rev + doc2 = self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc') + rev2 = doc2.rev + self.sync(self.db1, self.db2) + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + # if remote content is in conflicted state, then document revisions + # will be different. + # self.assertEqual(doc.rev, self.db2.get_doc('doc').rev) + v = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev1))) + self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev2))) + + def test_sync_autoresolves_moar(self): + """ + Test for sync autoresolve local. + + This test was adapted to decrypt remote content before assert. + """ + # here we test that when a database that has a conflicted document is + # the source of a sync, and the target database has a revision of the + # conflicted document that is newer than the source database's, and + # that target's database's document's content is the same as the + # source's document's conflict's, the source's document's conflict gets + # autoresolved, and the source's document's revision bumped. + # + # idea is as follows: + # A B + # a1 - + # `-------> + # a1 a1 + # v v + # a2 a1b1 + # `-------> + # a1b1+a2 a1b1 + # v + # a1b1+a2 a1b2 (a1b2 has same content as a2) + # `-------> + # a3b2 a1b2 (autoresolved) + # `-------> + # a3b2 a3b2 + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc') + self.sync(self.db1, self.db2) + for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: + doc = db.get_doc('doc') + doc.set_json(content) + db.put_doc(doc) + self.sync(self.db1, self.db2) + # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict + doc = self.db1.get_doc('doc') + rev1 = doc.rev + self.assertTrue(doc.has_conflicts) + # set db2 to have a doc of {} (same as db1 before the conflict) + doc = self.db2.get_doc('doc') + doc.set_json('{}') + self.db2.put_doc(doc) + rev2 = doc.rev + # sync it across + self.sync(self.db1, self.db2) + # tadaa! + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + vec1 = vectorclock.VectorClockRev(rev1) + vec2 = vectorclock.VectorClockRev(rev2) + vec3 = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(vec3.is_newer(vec1)) + self.assertTrue(vec3.is_newer(vec2)) + # because the conflict is on the source, sync it another time + self.sync(self.db1, self.db2) + # make sure db2 now has the exact same thing + doc1 = self.db1.get_doc('doc') + self.assertGetEncryptedDoc( + self.db2, + doc1.doc_id, doc1.rev, doc1.get_json(), False) + + def test_sync_autoresolves_moar_backwards(self): + # here we would test that when a database that has a conflicted + # document is the target of a sync, and the source database has a + # revision of the conflicted document that is newer than the target + # database's, and that source's database's document's content is the + # same as the target's document's conflict's, the target's document's + # conflict gets autoresolved, and the document's revision bumped. + # + # Despite that, in Soledad we suppose that the server never syncs, so + # it never has conflicted documents. Also, if it had, convergence + # would not be possible by checking document's contents because they + # would be encrypted in server. + # + # Therefore we suppress this test. + pass + + def test_sync_autoresolves_moar_backwards_three(self): + # here we would test that when a database that has a conflicted + # document is the target of a sync, and the source database has a + # revision of the conflicted document that is newer than the target + # database's, and that source's database's document's content is the + # same as the target's document's conflict's, the target's document's + # conflict gets autoresolved, and the document's revision bumped. + # + # We use the same reasoning from the last test to suppress this one. + pass + + def test_sync_pulling_doesnt_update_other_if_changed(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db2.create_doc_from_json(tests.simple_doc) + # After the local side has sent its list of docs, before we start + # receiving the "targets" response, we update the local database with a + # new record. + # When we finish synchronizing, we can notice that something locally + # was updated, and we cannot tell c2 our new updated generation + + def before_get_docs(state): + if state != 'before get_docs': + return + self.db1.create_doc_from_json(tests.simple_doc) + + self.assertEqual(0, self.sync(self.db1, self.db2, + trace_hook=before_get_docs)) + self.assertLastExchangeLog(self.db2, + {'receive': + {'docs': [], 'last_known_gen': 0}, + 'return': + {'docs': [(doc.doc_id, doc.rev)], + 'last_gen': 1}}) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + # c2 should not have gotten a '_record_sync_info' call, because the + # local database had been updated more than just by the messages + # returned from c2. + self.assertEqual( + (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) + + def test_sync_doesnt_update_other_if_nothing_pulled(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc) + + def no_record_sync_info(state): + if state != 'record_sync_info': + return + self.fail('SyncTarget.record_sync_info was called') + self.assertEqual(1, self.sync(self.db1, self.db2, + trace_hook_shallow=no_record_sync_info)) + self.assertEqual( + 1, + self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)[0]) + + def test_sync_ignores_convergence(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'both') + doc = self.db1.create_doc_from_json(tests.simple_doc) + self.db3 = self.create_database('test3', 'target') + self.assertEqual(1, self.sync(self.db1, self.db3)) + self.assertEqual(0, self.sync(self.db2, self.db3)) + self.assertEqual(1, self.sync(self.db1, self.db2)) + self.assertLastExchangeLog(self.db2, + {'receive': + {'docs': [(doc.doc_id, doc.rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 1}}) + + def test_sync_ignores_superseded(self): + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'both') + doc = self.db1.create_doc_from_json(tests.simple_doc) + doc_rev1 = doc.rev + self.db3 = self.create_database('test3', 'target') + self.sync(self.db1, self.db3) + self.sync(self.db2, self.db3) + new_content = '{"key": "altval"}' + doc.set_json(new_content) + self.db1.put_doc(doc) + doc_rev2 = doc.rev + self.sync(self.db2, self.db1) + self.assertLastExchangeLog(self.db1, + {'receive': + {'docs': [(doc.doc_id, doc_rev1)], + 'source_uid': 'test2', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': + {'docs': [(doc.doc_id, doc_rev2)], + 'last_gen': 2}}) + self.assertGetDoc(self.db1, doc.doc_id, doc_rev2, new_content, False) + + def test_sync_sees_remote_conflicted(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(tests.simple_doc) + doc_id = doc1.doc_id + doc1_rev = doc1.rev + self.db1.create_index('test-idx', 'key') + new_doc = '{"key": "altval"}' + doc2 = self.db2.create_doc_from_json(new_doc, doc_id=doc_id) + doc2_rev = doc2.rev + self.assertTransactionLog([doc1.doc_id], self.db1) + self.sync(self.db1, self.db2) + self.assertLastExchangeLog(self.db2, + {'receive': + {'docs': [(doc_id, doc1_rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': + {'docs': [(doc_id, doc2_rev)], + 'last_gen': 1}}) + self.assertTransactionLog([doc_id, doc_id], self.db1) + self.assertGetDoc(self.db1, doc_id, doc2_rev, new_doc, True) + self.assertGetDoc(self.db2, doc_id, doc2_rev, new_doc, False) + from_idx = self.db1.get_from_index('test-idx', 'altval')[0] + self.assertEqual(doc2.doc_id, from_idx.doc_id) + self.assertEqual(doc2.rev, from_idx.rev) + self.assertTrue(from_idx.has_conflicts) + self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) + + def test_sync_sees_remote_delete_conflicted(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(tests.simple_doc) + doc_id = doc1.doc_id + self.db1.create_index('test-idx', 'key') + self.sync(self.db1, self.db2) + doc2 = self.make_document(doc1.doc_id, doc1.rev, doc1.get_json()) + new_doc = '{"key": "altval"}' + doc1.set_json(new_doc) + self.db1.put_doc(doc1) + self.db2.delete_doc(doc2) + self.assertTransactionLog([doc_id, doc_id], self.db1) + self.sync(self.db1, self.db2) + self.assertLastExchangeLog(self.db2, + {'receive': + {'docs': [(doc_id, doc1.rev)], + 'source_uid': 'test1', + 'source_gen': 2, 'last_known_gen': 1}, + 'return': {'docs': [(doc_id, doc2.rev)], + 'last_gen': 2}}) + self.assertTransactionLog([doc_id, doc_id, doc_id], self.db1) + self.assertGetDocIncludeDeleted(self.db1, doc_id, doc2.rev, None, True) + self.assertGetDocIncludeDeleted( + self.db2, doc_id, doc2.rev, None, False) + self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) + + def test_sync_local_race_conflicted(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db1.create_doc_from_json(tests.simple_doc) + doc_id = doc.doc_id + doc1_rev = doc.rev + self.db1.create_index('test-idx', 'key') + self.sync(self.db1, self.db2) + content1 = '{"key": "localval"}' + content2 = '{"key": "altval"}' + doc.set_json(content2) + self.db2.put_doc(doc) + doc2_rev2 = doc.rev + triggered = [] + + def after_whatschanged(state): + if state != 'after whats_changed': + return + triggered.append(True) + doc = self.make_document(doc_id, doc1_rev, content1) + self.db1.put_doc(doc) + + self.sync(self.db1, self.db2, trace_hook=after_whatschanged) + self.assertEqual([True], triggered) + self.assertGetDoc(self.db1, doc_id, doc2_rev2, content2, True) + from_idx = self.db1.get_from_index('test-idx', 'altval')[0] + self.assertEqual(doc.doc_id, from_idx.doc_id) + self.assertEqual(doc.rev, from_idx.rev) + self.assertTrue(from_idx.has_conflicts) + self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) + self.assertEqual([], self.db1.get_from_index('test-idx', 'localval')) + + def test_sync_propagates_deletes(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'both') + doc1 = self.db1.create_doc_from_json(tests.simple_doc) + doc_id = doc1.doc_id + self.db1.create_index('test-idx', 'key') + self.sync(self.db1, self.db2) + self.db2.create_index('test-idx', 'key') + self.db3 = self.create_database('test3', 'target') + self.sync(self.db1, self.db3) + self.db1.delete_doc(doc1) + deleted_rev = doc1.rev + self.sync(self.db1, self.db2) + self.assertLastExchangeLog(self.db2, + {'receive': + {'docs': [(doc_id, deleted_rev)], + 'source_uid': 'test1', + 'source_gen': 2, 'last_known_gen': 1}, + 'return': {'docs': [], 'last_gen': 2}}) + self.assertGetDocIncludeDeleted( + self.db1, doc_id, deleted_rev, None, False) + self.assertGetDocIncludeDeleted( + self.db2, doc_id, deleted_rev, None, False) + self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) + self.assertEqual([], self.db2.get_from_index('test-idx', 'value')) + self.sync(self.db2, self.db3) + self.assertLastExchangeLog(self.db3, + {'receive': + {'docs': [(doc_id, deleted_rev)], + 'source_uid': 'test2', + 'source_gen': 2, + 'last_known_gen': 0}, + 'return': + {'docs': [], 'last_gen': 2}}) + self.assertGetDocIncludeDeleted( + self.db3, doc_id, deleted_rev, None, False) + + def test_sync_propagates_deletes_2(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json('{"a": "1"}', doc_id='the-doc') + self.sync(self.db1, self.db2) + doc1_2 = self.db2.get_doc('the-doc') + self.db2.delete_doc(doc1_2) + self.sync(self.db1, self.db2) + self.assertGetDocIncludeDeleted( + self.db1, 'the-doc', doc1_2.rev, None, False) + + def test_sync_detects_identical_replica_uid(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test1', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') + self.assertRaises( + errors.InvalidReplicaUID, self.sync, self.db1, self.db2) + + def test_optional_sync_preserve_json(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + cont1 = '{ "a": 2 }' + cont2 = '{ "b":3}' + self.db1.create_doc_from_json(cont1, doc_id="1") + self.db2.create_doc_from_json(cont2, doc_id="2") + self.sync(self.db1, self.db2) + self.assertEqual(cont1, self.db2.get_doc("1").get_json()) + self.assertEqual(cont2, self.db1.get_doc("2").get_json()) + + def test_sync_propagates_resolution(self): + """ + Test if synchronization propagates resolution. + + This test was adapted to decrypt remote content before assert. + """ + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'both') + doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') + db3 = self.create_database('test3', 'both') + self.sync(self.db2, self.db1) + self.assertEqual( + self.db1._get_generation_info(), + self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)) + self.assertEqual( + self.db2._get_generation_info(), + self.db1._get_replica_gen_and_trans_id(self.db2._replica_uid)) + self.sync(db3, self.db1) + # update on 2 + doc2 = self.make_document('the-doc', doc1.rev, '{"a": 2}') + self.db2.put_doc(doc2) + self.sync(self.db2, db3) + self.assertEqual(db3.get_doc('the-doc').rev, doc2.rev) + # update on 1 + doc1.set_json('{"a": 3}') + self.db1.put_doc(doc1) + # conflicts + self.sync(self.db2, self.db1) + self.sync(db3, self.db1) + self.assertTrue(self.db2.get_doc('the-doc').has_conflicts) + self.assertTrue(db3.get_doc('the-doc').has_conflicts) + # resolve + conflicts = self.db2.get_doc_conflicts('the-doc') + doc4 = self.make_document('the-doc', None, '{"a": 4}') + revs = [doc.rev for doc in conflicts] + self.db2.resolve_doc(doc4, revs) + doc2 = self.db2.get_doc('the-doc') + self.assertEqual(doc4.get_json(), doc2.get_json()) + self.assertFalse(doc2.has_conflicts) + self.sync(self.db2, db3) + doc3 = db3.get_doc('the-doc') + if ENC_SCHEME_KEY in doc3.content: + _crypto = self._soledad._crypto + key = _crypto.doc_passphrase(doc3.doc_id) + secret = _crypto.secret + doc3.set_json(decrypt_doc_dict( + doc3.content, + doc3.doc_id, doc3.rev, key, secret)) + self.assertEqual(doc4.get_json(), doc3.get_json()) + self.assertFalse(doc3.has_conflicts) + self.db1.close() + self.db2.close() + db3.close() + + def test_sync_puts_changes(self): + """ + Test if sync puts changes in remote replica. + + This test was adapted to decrypt remote content before assert. + """ + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db1.create_doc_from_json(tests.simple_doc) + self.assertEqual(1, self.sync(self.db1, self.db2)) + self.assertGetEncryptedDoc( + self.db2, doc.doc_id, doc.rev, tests.simple_doc, False) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [(doc.doc_id, doc.rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 1}}) + + def test_sync_pulls_changes(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db2.create_doc_from_json(tests.simple_doc) + self.db1.create_index('test-idx', 'key') + self.assertEqual(0, self.sync(self.db1, self.db2)) + self.assertGetDoc(self.db1, doc.doc_id, doc.rev, + tests.simple_doc, False) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) + self.assertLastExchangeLog(self.db2, + {'receive': + {'docs': [], 'last_known_gen': 0}, + 'return': + {'docs': [(doc.doc_id, doc.rev)], + 'last_gen': 1}}) + self.assertEqual([doc], self.db1.get_from_index('test-idx', 'value')) + + def test_sync_supersedes_conflicts(self): + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'target') + self.db3 = self.create_database('test3', 'both') + doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') + self.db2.create_doc_from_json('{"b": 1}', doc_id='the-doc') + self.db3.create_doc_from_json('{"c": 1}', doc_id='the-doc') + self.sync(self.db3, self.db1) + self.assertEqual( + self.db1._get_generation_info(), + self.db3._get_replica_gen_and_trans_id(self.db1._replica_uid)) + self.assertEqual( + self.db3._get_generation_info(), + self.db1._get_replica_gen_and_trans_id(self.db3._replica_uid)) + self.sync(self.db3, self.db2) + self.assertEqual( + self.db2._get_generation_info(), + self.db3._get_replica_gen_and_trans_id(self.db2._replica_uid)) + self.assertEqual( + self.db3._get_generation_info(), + self.db2._get_replica_gen_and_trans_id(self.db3._replica_uid)) + self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) + doc1.set_json('{"a": 2}') + self.db1.put_doc(doc1) + self.sync(self.db3, self.db1) + # original doc1 should have been removed from conflicts + self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) + + def test_sync_stops_after_get_sync_info(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc) + self.sync(self.db1, self.db2) + + def put_hook(state): + self.fail("Tracehook triggered for %s" % (state,)) + + self.sync(self.db1, self.db2, trace_hook_shallow=put_hook) + + def test_sync_detects_rollback_in_source(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') + self.sync(self.db1, self.db2) + self.db1_copy = self.copy_database(self.db1) + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.sync(self.db1, self.db2) + self.assertRaises( + errors.InvalidGeneration, self.sync, self.db1_copy, self.db2) + + def test_sync_detects_rollback_in_target(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.db2_copy = self.copy_database(self.db2) + self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.sync(self.db1, self.db2) + self.assertRaises( + errors.InvalidGeneration, self.sync, self.db1, self.db2_copy) + + def test_sync_detects_diverged_source(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db3 = self.copy_database(self.db1) + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.db3.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.assertRaises( + errors.InvalidTransactionId, self.sync, self.db3, self.db2) + + def test_sync_detects_diverged_target(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db3 = self.copy_database(self.db2) + self.db3.create_doc_from_json(tests.nested_doc, doc_id="divergent") + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.assertRaises( + errors.InvalidTransactionId, self.sync, self.db1, self.db3) + + def test_sync_detects_rollback_and_divergence_in_source(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') + self.sync(self.db1, self.db2) + self.db1_copy = self.copy_database(self.db1) + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.sync(self.db1, self.db2) + self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.assertRaises( + errors.InvalidTransactionId, self.sync, self.db1_copy, self.db2) + + def test_sync_detects_rollback_and_divergence_in_target(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.db2_copy = self.copy_database(self.db2) + self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.sync(self.db1, self.db2) + self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.assertRaises( + errors.InvalidTransactionId, self.sync, self.db1, self.db2_copy) + + +def make_local_db_and_soledad_target( + test, path='test', + source_replica_uid=uuid4().hex): + test.startTwistedServer() + replica_uid = os.path.basename(path) + db = test.request_state._create_database(replica_uid) + sync_db = test._soledad._sync_db + sync_enc_pool = test._soledad._sync_enc_pool + st = soledad_sync_target( + test, db._dbname, + source_replica_uid=source_replica_uid, + sync_db=sync_db, + sync_enc_pool=sync_enc_pool) + return db, st + +target_scenarios = [ + ('leap', { + 'create_db_and_target': make_local_db_and_soledad_target, + 'make_app_with_state': make_soledad_app, + 'do_sync': sync_via_synchronizer_and_soledad}), +] diff --git a/testing/tests/couch/__init__.py b/testing/tests/couch/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testing/tests/couch/couchdb.ini.template b/testing/tests/couch/couchdb.ini.template new file mode 100644 index 00000000..174d9d86 --- /dev/null +++ b/testing/tests/couch/couchdb.ini.template @@ -0,0 +1,22 @@ +; etc/couchdb/default.ini.tpl. Generated from default.ini.tpl.in by configure. + +; Upgrading CouchDB will overwrite this file. + +[couchdb] +database_dir = %(tempdir)s/lib +view_index_dir = %(tempdir)s/lib +max_document_size = 4294967296 ; 4 GB +os_process_timeout = 120000 ; 120 seconds. for view and external servers. +max_dbs_open = 100 +delayed_commits = true ; set this to false to ensure an fsync before 201 Created is returned +uri_file = %(tempdir)s/lib/couch.uri +file_compression = snappy + +[log] +file = %(tempdir)s/log/couch.log +level = info +include_sasl = true + +[httpd] +port = 0 +bind_address = 127.0.0.1 diff --git a/testing/tests/couch/test_couch.py b/testing/tests/couch/test_couch.py new file mode 100644 index 00000000..94c6ca92 --- /dev/null +++ b/testing/tests/couch/test_couch.py @@ -0,0 +1,1442 @@ +# -*- coding: utf-8 -*- +# test_couch.py +# Copyright (C) 2013-2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test ObjectStore and Couch backend bits. +""" +import json + +from uuid import uuid4 +from urlparse import urljoin + +from couchdb.client import Server + +from testscenarios import TestWithScenarios +from twisted.trial import unittest +from mock import Mock + +from leap.soledad.common.l2db import errors as u1db_errors +from leap.soledad.common.l2db import SyncTarget +from leap.soledad.common.l2db import vectorclock + +from leap.soledad.common import couch +from leap.soledad.common.document import ServerDocument +from leap.soledad.common.couch import errors + +from test_soledad import u1db_tests as tests +from test_soledad.util import CouchDBTestCase +from test_soledad.util import make_local_db_and_target +from test_soledad.util import sync_via_synchronizer + +from test_soledad.u1db_tests import test_backends +from test_soledad.u1db_tests import DatabaseBaseTests + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_common_backend`. +# ----------------------------------------------------------------------------- + +class TestCouchBackendImpl(CouchDBTestCase): + + def test__allocate_doc_id(self): + db = couch.CouchDatabase.open_database( + urljoin( + 'http://localhost:' + str(self.couch_port), + ('test-%s' % uuid4().hex) + ), + create=True, + ensure_ddocs=True) + doc_id1 = db._allocate_doc_id() + self.assertTrue(doc_id1.startswith('D-')) + self.assertEqual(34, len(doc_id1)) + int(doc_id1[len('D-'):], 16) + self.assertNotEqual(doc_id1, db._allocate_doc_id()) + self.delete_db(db._dbname) + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_backends`. +# ----------------------------------------------------------------------------- + +def make_couch_database_for_test(test, replica_uid): + port = str(test.couch_port) + dbname = ('test-%s' % uuid4().hex) + db = couch.CouchDatabase.open_database( + urljoin('http://localhost:' + port, dbname), + create=True, + replica_uid=replica_uid or 'test', + ensure_ddocs=True) + test.addCleanup(test.delete_db, dbname) + return db + + +def copy_couch_database_for_test(test, db): + port = str(test.couch_port) + couch_url = 'http://localhost:' + port + new_dbname = db._dbname + '_copy' + new_db = couch.CouchDatabase.open_database( + urljoin(couch_url, new_dbname), + create=True, + replica_uid=db._replica_uid or 'test') + # copy all docs + session = couch.Session() + old_couch_db = Server(couch_url, session=session)[db._dbname] + new_couch_db = Server(couch_url, session=session)[new_dbname] + for doc_id in old_couch_db: + doc = old_couch_db.get(doc_id) + # bypass u1db_config document + if doc_id == 'u1db_config': + pass + # copy design docs + elif doc_id.startswith('_design'): + del doc['_rev'] + new_couch_db.save(doc) + # copy u1db docs + elif 'u1db_rev' in doc: + new_doc = { + '_id': doc['_id'], + 'u1db_transactions': doc['u1db_transactions'], + 'u1db_rev': doc['u1db_rev'] + } + attachments = [] + if ('u1db_conflicts' in doc): + new_doc['u1db_conflicts'] = doc['u1db_conflicts'] + for c_rev in doc['u1db_conflicts']: + attachments.append('u1db_conflict_%s' % c_rev) + new_couch_db.save(new_doc) + # save conflict data + attachments.append('u1db_content') + for att_name in attachments: + att = old_couch_db.get_attachment(doc_id, att_name) + if (att is not None): + new_couch_db.put_attachment(new_doc, att, + filename=att_name) + # cleanup connections to prevent file descriptor leaking + return new_db + + +def make_document_for_test(test, doc_id, rev, content, has_conflicts=False): + return ServerDocument( + doc_id, rev, content, has_conflicts=has_conflicts) + + +COUCH_SCENARIOS = [ + ('couch', {'make_database_for_test': make_couch_database_for_test, + 'copy_database_for_test': copy_couch_database_for_test, + 'make_document_for_test': make_document_for_test, }), +] + + +class CouchTests( + TestWithScenarios, test_backends.AllDatabaseTests, CouchDBTestCase): + + scenarios = COUCH_SCENARIOS + + +class SoledadBackendTests( + TestWithScenarios, + test_backends.LocalDatabaseTests, + CouchDBTestCase): + + scenarios = COUCH_SCENARIOS + + +class CouchValidateGenNTransIdTests( + TestWithScenarios, + test_backends.LocalDatabaseValidateGenNTransIdTests, + CouchDBTestCase): + + scenarios = COUCH_SCENARIOS + + +class CouchValidateSourceGenTests( + TestWithScenarios, + test_backends.LocalDatabaseValidateSourceGenTests, + CouchDBTestCase): + + scenarios = COUCH_SCENARIOS + + +class CouchWithConflictsTests( + TestWithScenarios, + test_backends.LocalDatabaseWithConflictsTests, + CouchDBTestCase): + + scenarios = COUCH_SCENARIOS + + +# Notice: the CouchDB backend does not have indexing capabilities, so we do +# not test indexing now. + +# class CouchIndexTests(test_backends.DatabaseIndexTests, CouchDBTestCase): +# +# scenarios = COUCH_SCENARIOS +# +# def tearDown(self): +# self.db.delete_database() +# test_backends.DatabaseIndexTests.tearDown(self) + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_sync`. +# ----------------------------------------------------------------------------- + +target_scenarios = [ + ('local', {'create_db_and_target': make_local_db_and_target}), ] + + +simple_doc = tests.simple_doc +nested_doc = tests.nested_doc + + +class SoledadBackendSyncTargetTests( + TestWithScenarios, + DatabaseBaseTests, + CouchDBTestCase): + + # TODO: implement _set_trace_hook(_shallow) in CouchSyncTarget so + # skipped tests can be succesfully executed. + + # whitebox true means self.db is the actual local db object + # against which the sync is performed + whitebox = True + + scenarios = (tests.multiply_scenarios(COUCH_SCENARIOS, target_scenarios)) + + def set_trace_hook(self, callback, shallow=False): + setter = (self.st._set_trace_hook if not shallow else + self.st._set_trace_hook_shallow) + try: + setter(callback) + except NotImplementedError: + self.skipTest("%s does not implement _set_trace_hook" + % (self.st.__class__.__name__,)) + + def setUp(self): + CouchDBTestCase.setUp(self) + # other stuff + self.db, self.st = self.create_db_and_target(self) + self.other_changes = [] + + def tearDown(self): + self.db.close() + CouchDBTestCase.tearDown(self) + + def receive_doc(self, doc, gen, trans_id): + self.other_changes.append( + (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) + + def test_sync_exchange_returns_many_new_docs(self): + # This test was replicated to allow dictionaries to be compared after + # JSON expansion (because one dictionary may have many different + # serialized representations). + doc = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) + new_gen, _ = self.st.sync_exchange( + [], 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) + self.assertEqual(2, new_gen) + self.assertEqual( + [(doc.doc_id, doc.rev, json.loads(simple_doc), 1), + (doc2.doc_id, doc2.rev, json.loads(nested_doc), 2)], + [c[:-3] + (json.loads(c[-3]), c[-2]) for c in self.other_changes]) + if self.whitebox: + self.assertEqual( + self.db._last_exchange_log['return'], + {'last_gen': 2, 'docs': + [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]}) + + def test_get_sync_target(self): + self.assertIsNot(None, self.st) + + def test_get_sync_info(self): + self.assertEqual( + ('test', 0, '', 0, ''), self.st.get_sync_info('other')) + + def test_create_doc_updates_sync_info(self): + self.assertEqual( + ('test', 0, '', 0, ''), self.st.get_sync_info('other')) + self.db.create_doc_from_json(simple_doc) + self.assertEqual(1, self.st.get_sync_info('other')[1]) + + def test_record_sync_info(self): + self.st.record_sync_info('replica', 10, 'T-transid') + self.assertEqual( + ('test', 0, '', 10, 'T-transid'), self.st.get_sync_info('replica')) + + def test_sync_exchange(self): + docs_by_gen = [ + (self.make_document('doc-id', 'replica:1', simple_doc), 10, + 'T-sid')] + new_gen, trans_id = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertGetDoc(self.db, 'doc-id', 'replica:1', simple_doc, False) + self.assertTransactionLog(['doc-id'], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 1, last_trans_id), + (self.other_changes, new_gen, last_trans_id)) + self.assertEqual(10, self.st.get_sync_info('replica')[3]) + + def test_sync_exchange_deleted(self): + doc = self.db.create_doc_from_json('{}') + edit_rev = 'replica:1|' + doc.rev + docs_by_gen = [ + (self.make_document(doc.doc_id, edit_rev, None), 10, 'T-sid')] + new_gen, trans_id = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertGetDocIncludeDeleted( + self.db, doc.doc_id, edit_rev, None, False) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 2, last_trans_id), + (self.other_changes, new_gen, trans_id)) + self.assertEqual(10, self.st.get_sync_info('replica')[3]) + + def test_sync_exchange_push_many(self): + docs_by_gen = [ + (self.make_document('doc-id', 'replica:1', simple_doc), 10, 'T-1'), + (self.make_document('doc-id2', 'replica:1', nested_doc), 11, + 'T-2')] + new_gen, trans_id = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertGetDoc(self.db, 'doc-id', 'replica:1', simple_doc, False) + self.assertGetDoc(self.db, 'doc-id2', 'replica:1', nested_doc, False) + self.assertTransactionLog(['doc-id', 'doc-id2'], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 2, last_trans_id), + (self.other_changes, new_gen, trans_id)) + self.assertEqual(11, self.st.get_sync_info('replica')[3]) + + def test_sync_exchange_refuses_conflicts(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + (self.make_document(doc.doc_id, 'replica:1', new_doc), 10, + 'T-sid')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertEqual( + (doc.doc_id, doc.rev, simple_doc, 1), self.other_changes[0][:-1]) + self.assertEqual(1, new_gen) + if self.whitebox: + self.assertEqual(self.db._last_exchange_log['return'], + {'last_gen': 1, 'docs': [(doc.doc_id, doc.rev)]}) + + def test_sync_exchange_ignores_convergence(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + gen, txid = self.db._get_generation_info() + docs_by_gen = [ + (self.make_document(doc.doc_id, doc.rev, simple_doc), 10, 'T-sid')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=gen, + last_known_trans_id=txid, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertEqual(([], 1), (self.other_changes, new_gen)) + + def test_sync_exchange_returns_new_docs(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_gen, _ = self.st.sync_exchange( + [], 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertEqual( + (doc.doc_id, doc.rev, simple_doc, 1), self.other_changes[0][:-1]) + self.assertEqual(1, new_gen) + if self.whitebox: + self.assertEqual(self.db._last_exchange_log['return'], + {'last_gen': 1, 'docs': [(doc.doc_id, doc.rev)]}) + + def test_sync_exchange_returns_deleted_docs(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + new_gen, _ = self.st.sync_exchange( + [], 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + self.assertEqual( + (doc.doc_id, doc.rev, None, 2), self.other_changes[0][:-1]) + self.assertEqual(2, new_gen) + if self.whitebox: + self.assertEqual(self.db._last_exchange_log['return'], + {'last_gen': 2, 'docs': [(doc.doc_id, doc.rev)]}) + + def test_sync_exchange_getting_newer_docs(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + 'T-sid')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + self.assertEqual(([], 2), (self.other_changes, new_gen)) + + def test_sync_exchange_with_concurrent_updates_of_synced_doc(self): + expected = [] + + def before_whatschanged_cb(state): + if state != 'before whats_changed': + return + cont = '{"key": "cuncurrent"}' + conc_rev = self.db.put_doc( + self.make_document(doc.doc_id, 'test:1|z:2', cont)) + expected.append((doc.doc_id, conc_rev, cont, 3)) + + self.set_trace_hook(before_whatschanged_cb) + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + 'T-sid')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertEqual(expected, [c[:-1] for c in self.other_changes]) + self.assertEqual(3, new_gen) + + def test_sync_exchange_with_concurrent_updates(self): + + def after_whatschanged_cb(state): + if state != 'after whats_changed': + return + self.db.create_doc_from_json('{"new": "doc"}') + + self.set_trace_hook(after_whatschanged_cb) + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + 'T-sid')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertEqual(([], 2), (self.other_changes, new_gen)) + + def test_sync_exchange_converged_handling(self): + doc = self.db.create_doc_from_json(simple_doc) + docs_by_gen = [ + (self.make_document('new', 'other:1', '{}'), 4, 'T-foo'), + (self.make_document(doc.doc_id, doc.rev, doc.get_json()), 5, + 'T-bar')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertEqual(([], 2), (self.other_changes, new_gen)) + + def test_sync_exchange_detect_incomplete_exchange(self): + def before_get_docs_explode(state): + if state != 'before get_docs': + return + raise u1db_errors.U1DBError("fail") + self.set_trace_hook(before_get_docs_explode) + # suppress traceback printing in the wsgiref server + # self.patch(simple_server.ServerHandler, + # 'log_exception', lambda h, exc_info: None) + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertRaises( + (u1db_errors.U1DBError, u1db_errors.BrokenSyncStream), + self.st.sync_exchange, [], 'other-replica', + last_known_generation=0, last_known_trans_id=None, + return_doc_cb=self.receive_doc) + + def test_sync_exchange_doc_ids(self): + sync_exchange_doc_ids = getattr(self.st, 'sync_exchange_doc_ids', None) + if sync_exchange_doc_ids is None: + self.skipTest("sync_exchange_doc_ids not implemented") + db2 = self.create_database('test2') + doc = db2.create_doc_from_json(simple_doc) + new_gen, trans_id = sync_exchange_doc_ids( + db2, [(doc.doc_id, 10, 'T-sid')], 0, None, + return_doc_cb=self.receive_doc) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + self.assertTransactionLog([doc.doc_id], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 1, last_trans_id), + (self.other_changes, new_gen, trans_id)) + self.assertEqual(10, self.st.get_sync_info(db2._replica_uid)[3]) + + def test__set_trace_hook(self): + called = [] + + def cb(state): + called.append(state) + + self.set_trace_hook(cb) + self.st.sync_exchange([], 'replica', 0, None, self.receive_doc) + self.st.record_sync_info('replica', 0, 'T-sid') + self.assertEqual(['before whats_changed', + 'after whats_changed', + 'before get_docs', + 'record_sync_info', + ], + called) + + def test__set_trace_hook_shallow(self): + st_trace_shallow = self.st._set_trace_hook_shallow + target_st_trace_shallow = SyncTarget._set_trace_hook_shallow + same_meth = st_trace_shallow == self.st._set_trace_hook + same_fun = st_trace_shallow.im_func == target_st_trace_shallow.im_func + if (same_meth or same_fun): + # shallow same as full + expected = ['before whats_changed', + 'after whats_changed', + 'before get_docs', + 'record_sync_info', + ] + else: + expected = ['sync_exchange', 'record_sync_info'] + + called = [] + + def cb(state): + called.append(state) + + self.set_trace_hook(cb, shallow=True) + self.st.sync_exchange([], 'replica', 0, None, self.receive_doc) + self.st.record_sync_info('replica', 0, 'T-sid') + self.assertEqual(expected, called) + +sync_scenarios = [] +for name, scenario in COUCH_SCENARIOS: + scenario = dict(scenario) + scenario['do_sync'] = sync_via_synchronizer + sync_scenarios.append((name, scenario)) + scenario = dict(scenario) + + +class SoledadBackendSyncTests( + TestWithScenarios, + DatabaseBaseTests, + CouchDBTestCase): + + scenarios = sync_scenarios + + def create_database(self, replica_uid, sync_role=None): + if replica_uid == 'test' and sync_role is None: + # created up the chain by base class but unused + return None + db = self.create_database_for_role(replica_uid, sync_role) + if sync_role: + self._use_tracking[db] = (replica_uid, sync_role) + return db + + def create_database_for_role(self, replica_uid, sync_role): + # hook point for reuse + return DatabaseBaseTests.create_database(self, replica_uid) + + def copy_database(self, db, sync_role=None): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES + # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST + # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS + # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND + # NINJA TO YOUR HOUSE. + db_copy = self.copy_database_for_test(self, db) + name, orig_sync_role = self._use_tracking[db] + self._use_tracking[db_copy] = ( + name + '(copy)', sync_role or orig_sync_role) + return db_copy + + def sync(self, db_from, db_to, trace_hook=None, + trace_hook_shallow=None): + from_name, from_sync_role = self._use_tracking[db_from] + to_name, to_sync_role = self._use_tracking[db_to] + if from_sync_role not in ('source', 'both'): + raise Exception("%s marked for %s use but used as source" % + (from_name, from_sync_role)) + if to_sync_role not in ('target', 'both'): + raise Exception("%s marked for %s use but used as target" % + (to_name, to_sync_role)) + return self.do_sync(self, db_from, db_to, trace_hook, + trace_hook_shallow) + + def setUp(self): + self.db = None + self.db1 = None + self.db2 = None + self.db3 = None + self.db1_copy = None + self.db2_copy = None + self._use_tracking = {} + DatabaseBaseTests.setUp(self) + + def tearDown(self): + for db in [ + self.db, self.db1, self.db2, + self.db3, self.db1_copy, self.db2_copy + ]: + if db is not None: + self.delete_db(db._dbname) + db.close() + DatabaseBaseTests.tearDown(self) + + def assertLastExchangeLog(self, db, expected): + log = getattr(db, '_last_exchange_log', None) + if log is None: + return + self.assertEqual(expected, log) + + def test_sync_tracks_db_generation_of_other(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.assertEqual(0, self.sync(self.db1, self.db2)) + self.assertEqual( + (0, ''), self.db1._get_replica_gen_and_trans_id('test2')) + self.assertEqual( + (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [], 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 0}}) + + def test_sync_autoresolves(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(simple_doc, doc_id='doc') + rev1 = doc1.rev + doc2 = self.db2.create_doc_from_json(simple_doc, doc_id='doc') + rev2 = doc2.rev + self.sync(self.db1, self.db2) + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + self.assertEqual(doc.rev, self.db2.get_doc('doc').rev) + v = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev1))) + self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev2))) + + def test_sync_autoresolves_moar(self): + # here we test that when a database that has a conflicted document is + # the source of a sync, and the target database has a revision of the + # conflicted document that is newer than the source database's, and + # that target's database's document's content is the same as the + # source's document's conflict's, the source's document's conflict gets + # autoresolved, and the source's document's revision bumped. + # + # idea is as follows: + # A B + # a1 - + # `-------> + # a1 a1 + # v v + # a2 a1b1 + # `-------> + # a1b1+a2 a1b1 + # v + # a1b1+a2 a1b2 (a1b2 has same content as a2) + # `-------> + # a3b2 a1b2 (autoresolved) + # `-------> + # a3b2 a3b2 + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(simple_doc, doc_id='doc') + self.sync(self.db1, self.db2) + for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: + doc = db.get_doc('doc') + doc.set_json(content) + db.put_doc(doc) + self.sync(self.db1, self.db2) + # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict + doc = self.db1.get_doc('doc') + rev1 = doc.rev + self.assertTrue(doc.has_conflicts) + # set db2 to have a doc of {} (same as db1 before the conflict) + doc = self.db2.get_doc('doc') + doc.set_json('{}') + self.db2.put_doc(doc) + rev2 = doc.rev + # sync it across + self.sync(self.db1, self.db2) + # tadaa! + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + vec1 = vectorclock.VectorClockRev(rev1) + vec2 = vectorclock.VectorClockRev(rev2) + vec3 = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(vec3.is_newer(vec1)) + self.assertTrue(vec3.is_newer(vec2)) + # because the conflict is on the source, sync it another time + self.sync(self.db1, self.db2) + # make sure db2 now has the exact same thing + self.assertEqual(self.db1.get_doc('doc'), self.db2.get_doc('doc')) + + def test_sync_autoresolves_moar_backwards(self): + # here we test that when a database that has a conflicted document is + # the target of a sync, and the source database has a revision of the + # conflicted document that is newer than the target database's, and + # that source's database's document's content is the same as the + # target's document's conflict's, the target's document's conflict gets + # autoresolved, and the document's revision bumped. + # + # idea is as follows: + # A B + # a1 - + # `-------> + # a1 a1 + # v v + # a2 a1b1 + # `-------> + # a1b1+a2 a1b1 + # v + # a1b1+a2 a1b2 (a1b2 has same content as a2) + # <-------' + # a3b2 a3b2 (autoresolved and propagated) + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'both') + self.db1.create_doc_from_json(simple_doc, doc_id='doc') + self.sync(self.db1, self.db2) + for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: + doc = db.get_doc('doc') + doc.set_json(content) + db.put_doc(doc) + self.sync(self.db1, self.db2) + # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict + doc = self.db1.get_doc('doc') + rev1 = doc.rev + self.assertTrue(doc.has_conflicts) + revc = self.db1.get_doc_conflicts('doc')[-1].rev + # set db2 to have a doc of {} (same as db1 before the conflict) + doc = self.db2.get_doc('doc') + doc.set_json('{}') + self.db2.put_doc(doc) + rev2 = doc.rev + # sync it across + self.sync(self.db2, self.db1) + # tadaa! + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + vec1 = vectorclock.VectorClockRev(rev1) + vec2 = vectorclock.VectorClockRev(rev2) + vec3 = vectorclock.VectorClockRev(doc.rev) + vecc = vectorclock.VectorClockRev(revc) + self.assertTrue(vec3.is_newer(vec1)) + self.assertTrue(vec3.is_newer(vec2)) + self.assertTrue(vec3.is_newer(vecc)) + # make sure db2 now has the exact same thing + self.assertEqual(self.db1.get_doc('doc'), self.db2.get_doc('doc')) + + def test_sync_autoresolves_moar_backwards_three(self): + # same as autoresolves_moar_backwards, but with three databases (note + # all the syncs go in the same direction -- this is a more natural + # scenario): + # + # A B C + # a1 - - + # `-------> + # a1 a1 - + # `-------> + # a1 a1 a1 + # v v + # a2 a1b1 a1 + # `-------------------> + # a2 a1b1 a2 + # `-------> + # a2+a1b1 a2 + # v + # a2 a2+a1b1 a2c1 (same as a1b1) + # `-------------------> + # a2c1 a2+a1b1 a2c1 + # `-------> + # a2b2c1 a2b2c1 a2c1 + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'both') + self.db3 = self.create_database('test3', 'target') + self.db1.create_doc_from_json(simple_doc, doc_id='doc') + self.sync(self.db1, self.db2) + self.sync(self.db2, self.db3) + for db, content in [(self.db2, '{"hi": 42}'), + (self.db1, '{}'), + ]: + doc = db.get_doc('doc') + doc.set_json(content) + db.put_doc(doc) + self.sync(self.db1, self.db3) + self.sync(self.db2, self.db3) + # db2 and db3 now both have a doc of {}, but db2 has a + # conflict + doc = self.db2.get_doc('doc') + self.assertTrue(doc.has_conflicts) + revc = self.db2.get_doc_conflicts('doc')[-1].rev + self.assertEqual('{}', doc.get_json()) + self.assertEqual(self.db3.get_doc('doc').get_json(), doc.get_json()) + self.assertEqual(self.db3.get_doc('doc').rev, doc.rev) + # set db3 to have a doc of {hi:42} (same as db2 before the conflict) + doc = self.db3.get_doc('doc') + doc.set_json('{"hi": 42}') + self.db3.put_doc(doc) + rev3 = doc.rev + # sync it across to db1 + self.sync(self.db1, self.db3) + # db1 now has hi:42, with a rev that is newer than db2's doc + doc = self.db1.get_doc('doc') + rev1 = doc.rev + self.assertFalse(doc.has_conflicts) + self.assertEqual('{"hi": 42}', doc.get_json()) + VCR = vectorclock.VectorClockRev + self.assertTrue(VCR(rev1).is_newer(VCR(self.db2.get_doc('doc').rev))) + # so sync it to db2 + self.sync(self.db1, self.db2) + # tadaa! + doc = self.db2.get_doc('doc') + self.assertFalse(doc.has_conflicts) + # db2's revision of the document is strictly newer than db1's before + # the sync, and db3's before that sync way back when + self.assertTrue(VCR(doc.rev).is_newer(VCR(rev1))) + self.assertTrue(VCR(doc.rev).is_newer(VCR(rev3))) + self.assertTrue(VCR(doc.rev).is_newer(VCR(revc))) + # make sure both dbs now have the exact same thing + self.assertEqual(self.db1.get_doc('doc'), self.db2.get_doc('doc')) + + def test_sync_puts_changes(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db1.create_doc_from_json(simple_doc) + self.assertEqual(1, self.sync(self.db1, self.db2)) + self.assertGetDoc(self.db2, doc.doc_id, doc.rev, simple_doc, False) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [(doc.doc_id, doc.rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 1}}) + + def test_sync_pulls_changes(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db2.create_doc_from_json(simple_doc) + self.assertEqual(0, self.sync(self.db1, self.db2)) + self.assertGetDoc(self.db1, doc.doc_id, doc.rev, simple_doc, False) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [], 'last_known_gen': 0}, + 'return': {'docs': [(doc.doc_id, doc.rev)], + 'last_gen': 1}}) + self.assertGetDoc(self.db2, doc.doc_id, doc.rev, simple_doc, False) + + def test_sync_pulling_doesnt_update_other_if_changed(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db2.create_doc_from_json(simple_doc) + # After the local side has sent its list of docs, before we start + # receiving the "targets" response, we update the local database with a + # new record. + # When we finish synchronizing, we can notice that something locally + # was updated, and we cannot tell c2 our new updated generation + + def before_get_docs(state): + if state != 'before get_docs': + return + self.db1.create_doc_from_json(simple_doc) + + self.assertEqual(0, self.sync(self.db1, self.db2, + trace_hook=before_get_docs)) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [], 'last_known_gen': 0}, + 'return': {'docs': [(doc.doc_id, doc.rev)], + 'last_gen': 1}}) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + # c2 should not have gotten a '_record_sync_info' call, because the + # local database had been updated more than just by the messages + # returned from c2. + self.assertEqual( + (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) + + def test_sync_doesnt_update_other_if_nothing_pulled(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(simple_doc) + + def no_record_sync_info(state): + if state != 'record_sync_info': + return + self.fail('SyncTarget.record_sync_info was called') + self.assertEqual(1, self.sync(self.db1, self.db2, + trace_hook_shallow=no_record_sync_info)) + self.assertEqual( + 1, + self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)[0]) + + def test_sync_ignores_convergence(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'both') + doc = self.db1.create_doc_from_json(simple_doc) + self.db3 = self.create_database('test3', 'target') + self.assertEqual(1, self.sync(self.db1, self.db3)) + self.assertEqual(0, self.sync(self.db2, self.db3)) + self.assertEqual(1, self.sync(self.db1, self.db2)) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [(doc.doc_id, doc.rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 1}}) + + def test_sync_ignores_superseded(self): + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'both') + doc = self.db1.create_doc_from_json(simple_doc) + doc_rev1 = doc.rev + self.db3 = self.create_database('test3', 'target') + self.sync(self.db1, self.db3) + self.sync(self.db2, self.db3) + new_content = '{"key": "altval"}' + doc.set_json(new_content) + self.db1.put_doc(doc) + doc_rev2 = doc.rev + self.sync(self.db2, self.db1) + self.assertLastExchangeLog( + self.db1, + {'receive': {'docs': [(doc.doc_id, doc_rev1)], + 'source_uid': 'test2', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [(doc.doc_id, doc_rev2)], + 'last_gen': 2}}) + self.assertGetDoc(self.db1, doc.doc_id, doc_rev2, new_content, False) + + def test_sync_sees_remote_conflicted(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(simple_doc) + doc_id = doc1.doc_id + doc1_rev = doc1.rev + new_doc = '{"key": "altval"}' + doc2 = self.db2.create_doc_from_json(new_doc, doc_id=doc_id) + doc2_rev = doc2.rev + self.assertTransactionLog([doc1.doc_id], self.db1) + self.sync(self.db1, self.db2) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [(doc_id, doc1_rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [(doc_id, doc2_rev)], + 'last_gen': 1}}) + self.assertTransactionLog([doc_id, doc_id], self.db1) + self.assertGetDoc(self.db1, doc_id, doc2_rev, new_doc, True) + self.assertGetDoc(self.db2, doc_id, doc2_rev, new_doc, False) + + def test_sync_sees_remote_delete_conflicted(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(simple_doc) + doc_id = doc1.doc_id + self.sync(self.db1, self.db2) + doc2 = self.make_document(doc1.doc_id, doc1.rev, doc1.get_json()) + new_doc = '{"key": "altval"}' + doc1.set_json(new_doc) + self.db1.put_doc(doc1) + self.db2.delete_doc(doc2) + self.assertTransactionLog([doc_id, doc_id], self.db1) + self.sync(self.db1, self.db2) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [(doc_id, doc1.rev)], + 'source_uid': 'test1', + 'source_gen': 2, 'last_known_gen': 1}, + 'return': {'docs': [(doc_id, doc2.rev)], + 'last_gen': 2}}) + self.assertTransactionLog([doc_id, doc_id, doc_id], self.db1) + self.assertGetDocIncludeDeleted(self.db1, doc_id, doc2.rev, None, True) + self.assertGetDocIncludeDeleted( + self.db2, doc_id, doc2.rev, None, False) + + def test_sync_local_race_conflicted(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db1.create_doc_from_json(simple_doc) + doc_id = doc.doc_id + doc1_rev = doc.rev + self.sync(self.db1, self.db2) + content1 = '{"key": "localval"}' + content2 = '{"key": "altval"}' + doc.set_json(content2) + self.db2.put_doc(doc) + doc2_rev2 = doc.rev + triggered = [] + + def after_whatschanged(state): + if state != 'after whats_changed': + return + triggered.append(True) + doc = self.make_document(doc_id, doc1_rev, content1) + self.db1.put_doc(doc) + + self.sync(self.db1, self.db2, trace_hook=after_whatschanged) + self.assertEqual([True], triggered) + self.assertGetDoc(self.db1, doc_id, doc2_rev2, content2, True) + + def test_sync_propagates_deletes(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'both') + doc1 = self.db1.create_doc_from_json(simple_doc) + doc_id = doc1.doc_id + self.sync(self.db1, self.db2) + self.db3 = self.create_database('test3', 'target') + self.sync(self.db1, self.db3) + self.db1.delete_doc(doc1) + deleted_rev = doc1.rev + self.sync(self.db1, self.db2) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [(doc_id, deleted_rev)], + 'source_uid': 'test1', + 'source_gen': 2, 'last_known_gen': 1}, + 'return': {'docs': [], 'last_gen': 2}}) + self.assertGetDocIncludeDeleted( + self.db1, doc_id, deleted_rev, None, False) + self.assertGetDocIncludeDeleted( + self.db2, doc_id, deleted_rev, None, False) + self.sync(self.db2, self.db3) + self.assertLastExchangeLog( + self.db3, + {'receive': {'docs': [(doc_id, deleted_rev)], + 'source_uid': 'test2', + 'source_gen': 2, 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 2}}) + self.assertGetDocIncludeDeleted( + self.db3, doc_id, deleted_rev, None, False) + + def test_sync_propagates_deletes_2(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json('{"a": "1"}', doc_id='the-doc') + self.sync(self.db1, self.db2) + doc1_2 = self.db2.get_doc('the-doc') + self.db2.delete_doc(doc1_2) + self.sync(self.db1, self.db2) + self.assertGetDocIncludeDeleted( + self.db1, 'the-doc', doc1_2.rev, None, False) + + def test_sync_propagates_resolution(self): + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'both') + doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') + self.db3 = self.create_database('test3', 'both') + self.sync(self.db2, self.db1) + self.assertEqual( + self.db1._get_generation_info(), + self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)) + self.assertEqual( + self.db2._get_generation_info(), + self.db1._get_replica_gen_and_trans_id(self.db2._replica_uid)) + self.sync(self.db3, self.db1) + # update on 2 + doc2 = self.make_document('the-doc', doc1.rev, '{"a": 2}') + self.db2.put_doc(doc2) + self.sync(self.db2, self.db3) + self.assertEqual(self.db3.get_doc('the-doc').rev, doc2.rev) + # update on 1 + doc1.set_json('{"a": 3}') + self.db1.put_doc(doc1) + # conflicts + self.sync(self.db2, self.db1) + self.sync(self.db3, self.db1) + self.assertTrue(self.db2.get_doc('the-doc').has_conflicts) + self.assertTrue(self.db3.get_doc('the-doc').has_conflicts) + # resolve + conflicts = self.db2.get_doc_conflicts('the-doc') + doc4 = self.make_document('the-doc', None, '{"a": 4}') + revs = [doc.rev for doc in conflicts] + self.db2.resolve_doc(doc4, revs) + doc2 = self.db2.get_doc('the-doc') + self.assertEqual(doc4.get_json(), doc2.get_json()) + self.assertFalse(doc2.has_conflicts) + self.sync(self.db2, self.db3) + doc3 = self.db3.get_doc('the-doc') + self.assertEqual(doc4.get_json(), doc3.get_json()) + self.assertFalse(doc3.has_conflicts) + + def test_sync_supersedes_conflicts(self): + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'target') + self.db3 = self.create_database('test3', 'both') + doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') + self.db2.create_doc_from_json('{"b": 1}', doc_id='the-doc') + self.db3.create_doc_from_json('{"c": 1}', doc_id='the-doc') + self.sync(self.db3, self.db1) + self.assertEqual( + self.db1._get_generation_info(), + self.db3._get_replica_gen_and_trans_id(self.db1._replica_uid)) + self.assertEqual( + self.db3._get_generation_info(), + self.db1._get_replica_gen_and_trans_id(self.db3._replica_uid)) + self.sync(self.db3, self.db2) + self.assertEqual( + self.db2._get_generation_info(), + self.db3._get_replica_gen_and_trans_id(self.db2._replica_uid)) + self.assertEqual( + self.db3._get_generation_info(), + self.db2._get_replica_gen_and_trans_id(self.db3._replica_uid)) + self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) + doc1.set_json('{"a": 2}') + self.db1.put_doc(doc1) + self.sync(self.db3, self.db1) + # original doc1 should have been removed from conflicts + self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) + + def test_sync_stops_after_get_sync_info(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc) + self.sync(self.db1, self.db2) + + def put_hook(state): + self.fail("Tracehook triggered for %s" % (state,)) + + self.sync(self.db1, self.db2, trace_hook_shallow=put_hook) + + def test_sync_detects_identical_replica_uid(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test1', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') + self.assertRaises( + u1db_errors.InvalidReplicaUID, self.sync, self.db1, self.db2) + + def test_sync_detects_rollback_in_source(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') + self.sync(self.db1, self.db2) + self.db1_copy = self.copy_database(self.db1) + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.sync(self.db1, self.db2) + self.assertRaises( + u1db_errors.InvalidGeneration, self.sync, self.db1_copy, self.db2) + + def test_sync_detects_rollback_in_target(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.db2_copy = self.copy_database(self.db2) + self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.sync(self.db1, self.db2) + self.assertRaises( + u1db_errors.InvalidGeneration, self.sync, self.db1, self.db2_copy) + + def test_sync_detects_diverged_source(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db3 = self.copy_database(self.db1) + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.db3.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.assertRaises( + u1db_errors.InvalidTransactionId, self.sync, self.db3, self.db2) + + def test_sync_detects_diverged_target(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db3 = self.copy_database(self.db2) + self.db3.create_doc_from_json(tests.nested_doc, doc_id="divergent") + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.assertRaises( + u1db_errors.InvalidTransactionId, self.sync, self.db1, self.db3) + + def test_sync_detects_rollback_and_divergence_in_source(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') + self.sync(self.db1, self.db2) + self.db1_copy = self.copy_database(self.db1) + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.sync(self.db1, self.db2) + self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.assertRaises( + u1db_errors.InvalidTransactionId, self.sync, + self.db1_copy, self.db2) + + def test_sync_detects_rollback_and_divergence_in_target(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.db2_copy = self.copy_database(self.db2) + self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.sync(self.db1, self.db2) + self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.assertRaises( + u1db_errors.InvalidTransactionId, self.sync, + self.db1, self.db2_copy) + + def test_optional_sync_preserve_json(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + cont1 = '{"a": 2}' + cont2 = '{"b": 3}' + self.db1.create_doc_from_json(cont1, doc_id="1") + self.db2.create_doc_from_json(cont2, doc_id="2") + self.sync(self.db1, self.db2) + self.assertEqual(cont1, self.db2.get_doc("1").get_json()) + self.assertEqual(cont2, self.db1.get_doc("2").get_json()) + + +class SoledadBackendExceptionsTests(CouchDBTestCase): + + def setUp(self): + CouchDBTestCase.setUp(self) + + def create_db(self, ensure=True, dbname=None): + if not dbname: + dbname = ('test-%s' % uuid4().hex) + if dbname not in self.couch_server: + self.couch_server.create(dbname) + self.db = couch.CouchDatabase( + ('http://127.0.0.1:%d' % self.couch_port), + dbname, + ensure_ddocs=ensure) + + def tearDown(self): + self.db.delete_database() + self.db.close() + CouchDBTestCase.tearDown(self) + + def test_missing_design_doc_raises(self): + """ + Test that all methods that access design documents will raise if the + design docs are not present. + """ + self.create_db(ensure=False) + # get_generation_info() + self.assertRaises( + errors.MissingDesignDocError, + self.db.get_generation_info) + # get_trans_id_for_gen() + self.assertRaises( + errors.MissingDesignDocError, + self.db.get_trans_id_for_gen, 1) + # get_transaction_log() + self.assertRaises( + errors.MissingDesignDocError, + self.db.get_transaction_log) + # whats_changed() + self.assertRaises( + errors.MissingDesignDocError, + self.db.whats_changed) + + def test_missing_design_doc_functions_raises(self): + """ + Test that all methods that access design documents list functions + will raise if the functions are not present. + """ + self.create_db(ensure=True) + # erase views from _design/transactions + transactions = self.db._database['_design/transactions'] + transactions['lists'] = {} + self.db._database.save(transactions) + # get_generation_info() + self.assertRaises( + errors.MissingDesignDocListFunctionError, + self.db.get_generation_info) + # get_trans_id_for_gen() + self.assertRaises( + errors.MissingDesignDocListFunctionError, + self.db.get_trans_id_for_gen, 1) + # whats_changed() + self.assertRaises( + errors.MissingDesignDocListFunctionError, + self.db.whats_changed) + + def test_absent_design_doc_functions_raises(self): + """ + Test that all methods that access design documents list functions + will raise if the functions are not present. + """ + self.create_db(ensure=True) + # erase views from _design/transactions + transactions = self.db._database['_design/transactions'] + del transactions['lists'] + self.db._database.save(transactions) + # get_generation_info() + self.assertRaises( + errors.MissingDesignDocListFunctionError, + self.db.get_generation_info) + # _get_trans_id_for_gen() + self.assertRaises( + errors.MissingDesignDocListFunctionError, + self.db.get_trans_id_for_gen, 1) + # whats_changed() + self.assertRaises( + errors.MissingDesignDocListFunctionError, + self.db.whats_changed) + + def test_missing_design_doc_named_views_raises(self): + """ + Test that all methods that access design documents' named views will + raise if the views are not present. + """ + self.create_db(ensure=True) + # erase views from _design/docs + docs = self.db._database['_design/docs'] + del docs['views'] + self.db._database.save(docs) + # erase views from _design/syncs + syncs = self.db._database['_design/syncs'] + del syncs['views'] + self.db._database.save(syncs) + # erase views from _design/transactions + transactions = self.db._database['_design/transactions'] + del transactions['views'] + self.db._database.save(transactions) + # get_generation_info() + self.assertRaises( + errors.MissingDesignDocNamedViewError, + self.db.get_generation_info) + # _get_trans_id_for_gen() + self.assertRaises( + errors.MissingDesignDocNamedViewError, + self.db.get_trans_id_for_gen, 1) + # _get_transaction_log() + self.assertRaises( + errors.MissingDesignDocNamedViewError, + self.db.get_transaction_log) + # whats_changed() + self.assertRaises( + errors.MissingDesignDocNamedViewError, + self.db.whats_changed) + + def test_deleted_design_doc_raises(self): + """ + Test that all methods that access design documents will raise if the + design docs are not present. + """ + self.create_db(ensure=True) + # delete _design/docs + del self.db._database['_design/docs'] + # delete _design/syncs + del self.db._database['_design/syncs'] + # delete _design/transactions + del self.db._database['_design/transactions'] + # get_generation_info() + self.assertRaises( + errors.MissingDesignDocDeletedError, + self.db.get_generation_info) + # get_trans_id_for_gen() + self.assertRaises( + errors.MissingDesignDocDeletedError, + self.db.get_trans_id_for_gen, 1) + # get_transaction_log() + self.assertRaises( + errors.MissingDesignDocDeletedError, + self.db.get_transaction_log) + # whats_changed() + self.assertRaises( + errors.MissingDesignDocDeletedError, + self.db.whats_changed) + + def test_ensure_ddoc_independently(self): + """ + Test that a missing ddocs other than _design/docs will be ensured + even if _design/docs is there. + """ + self.create_db(ensure=True) + del self.db._database['_design/transactions'] + self.assertRaises( + errors.MissingDesignDocDeletedError, + self.db.get_transaction_log) + self.create_db(ensure=True, dbname=self.db._dbname) + self.db.get_transaction_log() + + def test_ensure_security_doc(self): + """ + Ensure_security creates a _security ddoc to ensure that only soledad + will have the lowest privileged access to an user db. + """ + self.create_db(ensure=False) + self.assertFalse(self.db._database.resource.get_json('_security')[2]) + self.db.ensure_security_ddoc() + security_ddoc = self.db._database.resource.get_json('_security')[2] + self.assertIn('admins', security_ddoc) + self.assertFalse(security_ddoc['admins']['names']) + self.assertIn('members', security_ddoc) + self.assertIn('soledad', security_ddoc['members']['names']) + + def test_ensure_security_from_configuration(self): + """ + Given a configuration, follow it to create the security document + """ + self.create_db(ensure=False) + configuration = {'members': ['user1', 'user2'], + 'members_roles': ['role1', 'role2'], + 'admins': ['admin'], + 'admins_roles': ['administrators'] + } + self.db.ensure_security_ddoc(configuration) + + security_ddoc = self.db._database.resource.get_json('_security')[2] + self.assertEquals(configuration['admins'], + security_ddoc['admins']['names']) + self.assertEquals(configuration['admins_roles'], + security_ddoc['admins']['roles']) + self.assertEquals(configuration['members'], + security_ddoc['members']['names']) + self.assertEquals(configuration['members_roles'], + security_ddoc['members']['roles']) + + +class DatabaseNameValidationTest(unittest.TestCase): + + def test_database_name_validation(self): + inject = couch.state.is_db_name_valid("user-deadbeef | cat /secret") + self.assertFalse(inject) + self.assertTrue(couch.state.is_db_name_valid("user-cafe1337")) + + +class CommandBasedDBCreationTest(unittest.TestCase): + + def test_ensure_db_using_custom_command(self): + state = couch.state.CouchServerState("url", create_cmd="echo") + mock_db = Mock() + mock_db.replica_uid = 'replica_uid' + state.open_database = Mock(return_value=mock_db) + db, replica_uid = state.ensure_database("user-1337") # works + self.assertEquals(mock_db, db) + self.assertEquals(mock_db.replica_uid, replica_uid) + + def test_raises_unauthorized_on_failure(self): + state = couch.state.CouchServerState("url", create_cmd="inexistent") + self.assertRaises(u1db_errors.Unauthorized, + state.ensure_database, "user-1337") + + def test_raises_unauthorized_by_default(self): + state = couch.state.CouchServerState("url") + self.assertRaises(u1db_errors.Unauthorized, + state.ensure_database, "user-1337") diff --git a/testing/tests/couch/test_couch_operations_atomicity.py b/testing/tests/couch/test_couch_operations_atomicity.py new file mode 100644 index 00000000..aec9c6cf --- /dev/null +++ b/testing/tests/couch/test_couch_operations_atomicity.py @@ -0,0 +1,371 @@ +# -*- coding: utf-8 -*- +# test_couch_operations_atomicity.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test atomicity of couch operations. +""" +import os +import tempfile +import threading + +from urlparse import urljoin +from twisted.internet import defer +from uuid import uuid4 + +from leap.soledad.client import Soledad +from leap.soledad.common.couch.state import CouchServerState +from leap.soledad.common.couch import CouchDatabase + +from test_soledad.util import ( + make_token_soledad_app, + make_soledad_document_for_test, + soledad_sync_target, +) +from test_soledad.util import CouchDBTestCase +from test_soledad.u1db_tests import TestCaseWithServer + + +REPEAT_TIMES = 20 + + +class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): + + @staticmethod + def make_app_after_state(state): + return make_token_soledad_app(state) + + make_document_for_test = make_soledad_document_for_test + + sync_target = soledad_sync_target + + def _soledad_instance(self, user=None, passphrase=u'123', + prefix='', + secrets_path='secrets.json', + local_db_path='soledad.u1db', server_url='', + cert_file=None, auth_token=None): + """ + Instantiate Soledad. + """ + user = user or self.user + + # this callback ensures we save a document which is sent to the shared + # db. + def _put_doc_side_effect(doc): + self._doc_put = doc + + soledad = Soledad( + user, + passphrase, + secrets_path=os.path.join(self.tempdir, prefix, secrets_path), + local_db_path=os.path.join( + self.tempdir, prefix, local_db_path), + server_url=server_url, + cert_file=cert_file, + auth_token=auth_token, + shared_db=self.get_default_shared_mock(_put_doc_side_effect)) + self.addCleanup(soledad.close) + return soledad + + def make_app(self): + self.request_state = CouchServerState(self.couch_url) + return self.make_app_after_state(self.request_state) + + def setUp(self): + TestCaseWithServer.setUp(self) + CouchDBTestCase.setUp(self) + self.user = ('user-%s' % uuid4().hex) + self.db = CouchDatabase.open_database( + urljoin(self.couch_url, 'user-' + self.user), + create=True, + replica_uid='replica', + ensure_ddocs=True) + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + self.startTwistedServer() + + def tearDown(self): + self.db.delete_database() + self.db.close() + CouchDBTestCase.tearDown(self) + TestCaseWithServer.tearDown(self) + + # + # Sequential tests + # + + def test_correct_transaction_log_after_sequential_puts(self): + """ + Assert that the transaction_log increases accordingly with sequential + puts. + """ + doc = self.db.create_doc({'ops': 0}) + docs = [doc.doc_id] + for i in range(0, REPEAT_TIMES): + self.assertEqual( + i + 1, len(self.db._get_transaction_log())) + doc.content['ops'] += 1 + self.db.put_doc(doc) + docs.append(doc.doc_id) + + # assert length of transaction_log + transaction_log = self.db._get_transaction_log() + self.assertEqual( + REPEAT_TIMES + 1, len(transaction_log)) + + # assert that all entries in the log belong to the same doc + self.assertEqual(REPEAT_TIMES + 1, len(docs)) + for doc_id in docs: + self.assertEqual( + REPEAT_TIMES + 1, + len(filter(lambda t: t[0] == doc_id, transaction_log))) + + def test_correct_transaction_log_after_sequential_deletes(self): + """ + Assert that the transaction_log increases accordingly with sequential + puts and deletes. + """ + docs = [] + for i in range(0, REPEAT_TIMES): + doc = self.db.create_doc({'ops': 0}) + self.assertEqual( + 2 * i + 1, len(self.db._get_transaction_log())) + docs.append(doc.doc_id) + self.db.delete_doc(doc) + self.assertEqual( + 2 * i + 2, len(self.db._get_transaction_log())) + + # assert length of transaction_log + transaction_log = self.db._get_transaction_log() + self.assertEqual( + 2 * REPEAT_TIMES, len(transaction_log)) + + # assert that each doc appears twice in the transaction_log + self.assertEqual(REPEAT_TIMES, len(docs)) + for doc_id in docs: + self.assertEqual( + 2, + len(filter(lambda t: t[0] == doc_id, transaction_log))) + + @defer.inlineCallbacks + def test_correct_sync_log_after_sequential_syncs(self): + """ + Assert that the sync_log increases accordingly with sequential syncs. + """ + sol = self._soledad_instance( + auth_token='auth-token', + server_url=self.getURL()) + source_replica_uid = sol._dbpool.replica_uid + + def _create_docs(): + deferreds = [] + for i in xrange(0, REPEAT_TIMES): + deferreds.append(sol.create_doc({})) + return defer.gatherResults(deferreds) + + def _assert_transaction_and_sync_logs(results, sync_idx): + # assert sizes of transaction and sync logs + self.assertEqual( + sync_idx * REPEAT_TIMES, + len(self.db._get_transaction_log())) + gen, _ = self.db._get_replica_gen_and_trans_id(source_replica_uid) + self.assertEqual(sync_idx * REPEAT_TIMES, gen) + + def _assert_sync(results, sync_idx): + gen, docs = results + self.assertEqual((sync_idx + 1) * REPEAT_TIMES, gen) + self.assertEqual((sync_idx + 1) * REPEAT_TIMES, len(docs)) + # assert sizes of transaction and sync logs + self.assertEqual((sync_idx + 1) * REPEAT_TIMES, + len(self.db._get_transaction_log())) + target_known_gen, target_known_trans_id = \ + self.db._get_replica_gen_and_trans_id(source_replica_uid) + # assert it has the correct gen and trans_id + conn_key = sol._dbpool._u1dbconnections.keys().pop() + conn = sol._dbpool._u1dbconnections[conn_key] + sol_gen, sol_trans_id = conn._get_generation_info() + self.assertEqual(sol_gen, target_known_gen) + self.assertEqual(sol_trans_id, target_known_trans_id) + + # sync first time and assert success + results = yield _create_docs() + _assert_transaction_and_sync_logs(results, 0) + yield sol.sync() + results = yield sol.get_all_docs() + _assert_sync(results, 0) + + # create more docs, sync second time and assert success + results = yield _create_docs() + _assert_transaction_and_sync_logs(results, 1) + yield sol.sync() + results = yield sol.get_all_docs() + _assert_sync(results, 1) + + # + # Concurrency tests + # + + class _WorkerThread(threading.Thread): + + def __init__(self, params, run_method): + threading.Thread.__init__(self) + self._params = params + self._run_method = run_method + + def run(self): + self._run_method(self) + + def test_correct_transaction_log_after_concurrent_puts(self): + """ + Assert that the transaction_log increases accordingly with concurrent + puts. + """ + pool = threading.BoundedSemaphore(value=1) + threads = [] + docs = [] + + def _run_method(self): + doc = self._params['db'].create_doc({}) + pool.acquire() + self._params['docs'].append(doc.doc_id) + pool.release() + + for i in range(0, REPEAT_TIMES): + thread = self._WorkerThread( + {'docs': docs, 'db': self.db}, + _run_method) + thread.start() + threads.append(thread) + + for thread in threads: + thread.join() + + # assert length of transaction_log + transaction_log = self.db._get_transaction_log() + self.assertEqual( + REPEAT_TIMES, len(transaction_log)) + + # assert all documents are in the log + self.assertEqual(REPEAT_TIMES, len(docs)) + for doc_id in docs: + self.assertEqual( + 1, + len(filter(lambda t: t[0] == doc_id, transaction_log))) + + def test_correct_transaction_log_after_concurrent_deletes(self): + """ + Assert that the transaction_log increases accordingly with concurrent + puts and deletes. + """ + threads = [] + docs = [] + pool = threading.BoundedSemaphore(value=1) + + # create/delete method that will be run concurrently + def _run_method(self): + doc = self._params['db'].create_doc({}) + pool.acquire() + docs.append(doc.doc_id) + pool.release() + self._params['db'].delete_doc(doc) + + # launch concurrent threads + for i in range(0, REPEAT_TIMES): + thread = self._WorkerThread({'db': self.db}, _run_method) + thread.start() + threads.append(thread) + + # wait for threads to finish + for thread in threads: + thread.join() + + # assert transaction log + transaction_log = self.db._get_transaction_log() + self.assertEqual( + 2 * REPEAT_TIMES, len(transaction_log)) + # assert that each doc appears twice in the transaction_log + self.assertEqual(REPEAT_TIMES, len(docs)) + for doc_id in docs: + self.assertEqual( + 2, + len(filter(lambda t: t[0] == doc_id, transaction_log))) + + def test_correct_sync_log_after_concurrent_puts_and_sync(self): + """ + Assert that the sync_log is correct after concurrent syncs. + """ + docs = [] + + sol = self._soledad_instance( + auth_token='auth-token', + server_url=self.getURL()) + + def _save_doc_ids(results): + for doc in results: + docs.append(doc.doc_id) + + # create documents in parallel + deferreds = [] + for i in range(0, REPEAT_TIMES): + d = sol.create_doc({}) + deferreds.append(d) + + # wait for documents creation and sync + d = defer.gatherResults(deferreds) + d.addCallback(_save_doc_ids) + d.addCallback(lambda _: sol.sync()) + + def _assert_logs(results): + transaction_log = self.db._get_transaction_log() + self.assertEqual(REPEAT_TIMES, len(transaction_log)) + # assert all documents are in the remote log + self.assertEqual(REPEAT_TIMES, len(docs)) + for doc_id in docs: + self.assertEqual( + 1, + len(filter(lambda t: t[0] == doc_id, transaction_log))) + + d.addCallback(_assert_logs) + d.addCallback(lambda _: sol.close()) + + return d + + @defer.inlineCallbacks + def test_concurrent_syncs_do_not_fail(self): + """ + Assert that concurrent attempts to sync end up being executed + sequentially and do not fail. + """ + docs = [] + + sol = self._soledad_instance( + auth_token='auth-token', + server_url=self.getURL()) + + deferreds = [] + for i in xrange(0, REPEAT_TIMES): + d = sol.create_doc({}) + d.addCallback(lambda doc: docs.append(doc.doc_id)) + d.addCallback(lambda _: sol.sync()) + deferreds.append(d) + yield defer.gatherResults(deferreds, consumeErrors=True) + + transaction_log = self.db._get_transaction_log() + self.assertEqual(REPEAT_TIMES, len(transaction_log)) + # assert all documents are in the remote log + self.assertEqual(REPEAT_TIMES, len(docs)) + for doc_id in docs: + self.assertEqual( + 1, + len(filter(lambda t: t[0] == doc_id, transaction_log))) diff --git a/testing/tests/server/__init__.py b/testing/tests/server/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testing/tests/server/test_server.py b/testing/tests/server/test_server.py new file mode 100644 index 00000000..b99d1939 --- /dev/null +++ b/testing/tests/server/test_server.py @@ -0,0 +1,536 @@ +# -*- coding: utf-8 -*- +# test_server.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for server-related functionality. +""" +import binascii +import mock +import os +import tempfile + +from hashlib import sha512 +from pkg_resources import resource_filename +from urlparse import urljoin +from uuid import uuid4 + +from twisted.internet import defer +from twisted.trial import unittest + +from leap.soledad.common.couch.state import CouchServerState +from leap.soledad.common.couch import CouchDatabase +from test_soledad.u1db_tests import TestCaseWithServer +from test_soledad.util import CouchDBTestCase +from test_soledad.util import ( + make_token_soledad_app, + make_soledad_document_for_test, + soledad_sync_target, + BaseSoledadTest, +) + +from leap.soledad.common import crypto +from leap.soledad.client import Soledad +from leap.soledad.server import load_configuration +from leap.soledad.server import CONFIG_DEFAULTS +from leap.soledad.server.auth import URLToAuthorization +from leap.soledad.server.auth import SoledadTokenAuthMiddleware + + +class ServerAuthenticationMiddlewareTestCase(CouchDBTestCase): + + def setUp(self): + super(ServerAuthenticationMiddlewareTestCase, self).setUp() + app = mock.Mock() + self._state = CouchServerState(self.couch_url) + app.state = self._state + self.auth_middleware = SoledadTokenAuthMiddleware(app) + self._authorize('valid-uuid', 'valid-token') + + def _authorize(self, uuid, token): + token_doc = {} + token_doc['_id'] = sha512(token).hexdigest() + token_doc[self._state.TOKENS_USER_ID_KEY] = uuid + token_doc[self._state.TOKENS_TYPE_KEY] = \ + self._state.TOKENS_TYPE_DEF + dbname = self._state._tokens_dbname() + db = self.couch_server.create(dbname) + db.save(token_doc) + self.addCleanup(self.delete_db, db.name) + + def test_authorized_user(self): + is_authorized = self.auth_middleware._verify_authentication_data + self.assertTrue(is_authorized('valid-uuid', 'valid-token')) + self.assertFalse(is_authorized('valid-uuid', 'invalid-token')) + self.assertFalse(is_authorized('invalid-uuid', 'valid-token')) + self.assertFalse(is_authorized('eve', 'invalid-token')) + + +class ServerAuthorizationTestCase(BaseSoledadTest): + + """ + Tests related to Soledad server authorization. + """ + + def setUp(self): + pass + + def tearDown(self): + pass + + def _make_environ(self, path_info, request_method): + return { + 'PATH_INFO': path_info, + 'REQUEST_METHOD': request_method, + } + + def test_verify_action_with_correct_dbnames(self): + """ + Test encrypting and decrypting documents. + + The following table lists the authorized actions among all possible + u1db remote actions: + + URL path | Authorized actions + -------------------------------------------------- + / | GET + /shared-db | GET + /shared-db/docs | - + /shared-db/doc/{id} | GET, PUT, DELETE + /shared-db/sync-from/{source} | - + /user-db | GET, PUT, DELETE + /user-db/docs | - + /user-db/doc/{id} | - + /user-db/sync-from/{source} | GET, PUT, POST + """ + uuid = uuid4().hex + authmap = URLToAuthorization(uuid,) + dbname = authmap._user_db_name + # test global auth + self.assertTrue( + authmap.is_authorized(self._make_environ('/', 'GET'))) + # test shared-db database resource auth + self.assertTrue( + authmap.is_authorized( + self._make_environ('/shared', 'GET'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/shared', 'PUT'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/shared', 'DELETE'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/shared', 'POST'))) + # test shared-db docs resource auth + self.assertFalse( + authmap.is_authorized( + self._make_environ('/shared/docs', 'GET'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/shared/docs', 'PUT'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/shared/docs', 'DELETE'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/shared/docs', 'POST'))) + # test shared-db doc resource auth + self.assertTrue( + authmap.is_authorized( + self._make_environ('/shared/doc/x', 'GET'))) + self.assertTrue( + authmap.is_authorized( + self._make_environ('/shared/doc/x', 'PUT'))) + self.assertTrue( + authmap.is_authorized( + self._make_environ('/shared/doc/x', 'DELETE'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/shared/doc/x', 'POST'))) + # test shared-db sync resource auth + self.assertFalse( + authmap.is_authorized( + self._make_environ('/shared/sync-from/x', 'GET'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/shared/sync-from/x', 'PUT'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/shared/sync-from/x', 'DELETE'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/shared/sync-from/x', 'POST'))) + # test user-db database resource auth + self.assertTrue( + authmap.is_authorized( + self._make_environ('/%s' % dbname, 'GET'))) + self.assertTrue( + authmap.is_authorized( + self._make_environ('/%s' % dbname, 'PUT'))) + self.assertTrue( + authmap.is_authorized( + self._make_environ('/%s' % dbname, 'DELETE'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s' % dbname, 'POST'))) + # test user-db docs resource auth + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s/docs' % dbname, 'GET'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s/docs' % dbname, 'PUT'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s/docs' % dbname, 'DELETE'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s/docs' % dbname, 'POST'))) + # test user-db doc resource auth + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s/doc/x' % dbname, 'GET'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s/doc/x' % dbname, 'PUT'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s/doc/x' % dbname, 'DELETE'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s/doc/x' % dbname, 'POST'))) + # test user-db sync resource auth + self.assertTrue( + authmap.is_authorized( + self._make_environ('/%s/sync-from/x' % dbname, 'GET'))) + self.assertTrue( + authmap.is_authorized( + self._make_environ('/%s/sync-from/x' % dbname, 'PUT'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s/sync-from/x' % dbname, 'DELETE'))) + self.assertTrue( + authmap.is_authorized( + self._make_environ('/%s/sync-from/x' % dbname, 'POST'))) + + def test_verify_action_with_wrong_dbnames(self): + """ + Test if authorization fails for a wrong dbname. + """ + uuid = uuid4().hex + authmap = URLToAuthorization(uuid) + dbname = 'somedb' + # test wrong-db database resource auth + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s' % dbname, 'GET'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s' % dbname, 'PUT'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s' % dbname, 'DELETE'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s' % dbname, 'POST'))) + # test wrong-db docs resource auth + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s/docs' % dbname, 'GET'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s/docs' % dbname, 'PUT'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s/docs' % dbname, 'DELETE'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s/docs' % dbname, 'POST'))) + # test wrong-db doc resource auth + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s/doc/x' % dbname, 'GET'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s/doc/x' % dbname, 'PUT'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s/doc/x' % dbname, 'DELETE'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s/doc/x' % dbname, 'POST'))) + # test wrong-db sync resource auth + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s/sync-from/x' % dbname, 'GET'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s/sync-from/x' % dbname, 'PUT'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s/sync-from/x' % dbname, 'DELETE'))) + self.assertFalse( + authmap.is_authorized( + self._make_environ('/%s/sync-from/x' % dbname, 'POST'))) + + +class EncryptedSyncTestCase( + CouchDBTestCase, TestCaseWithServer): + + """ + Tests for encrypted sync using Soledad server backed by a couch database. + """ + + # increase twisted.trial's timeout because large files syncing might take + # some time to finish. + timeout = 500 + + @staticmethod + def make_app_with_state(state): + return make_token_soledad_app(state) + + make_document_for_test = make_soledad_document_for_test + + sync_target = soledad_sync_target + + def _soledad_instance(self, user=None, passphrase=u'123', + prefix='', + secrets_path='secrets.json', + local_db_path='soledad.u1db', + server_url='', + cert_file=None, auth_token=None): + """ + Instantiate Soledad. + """ + + # this callback ensures we save a document which is sent to the shared + # db. + def _put_doc_side_effect(doc): + self._doc_put = doc + + if not server_url: + # attempt to find the soledad server url + server_address = None + server = getattr(self, 'server', None) + if server: + server_address = getattr(self.server, 'server_address', None) + else: + host = self.port.getHost() + server_address = (host.host, host.port) + if server_address: + server_url = 'http://%s:%d' % (server_address) + + return Soledad( + user, + passphrase, + secrets_path=os.path.join(self.tempdir, prefix, secrets_path), + local_db_path=os.path.join( + self.tempdir, prefix, local_db_path), + server_url=server_url, + cert_file=cert_file, + auth_token=auth_token, + shared_db=self.get_default_shared_mock(_put_doc_side_effect)) + + def make_app(self): + self.request_state = CouchServerState(self.couch_url) + return self.make_app_with_state(self.request_state) + + def setUp(self): + # the order of the following initializations is crucial because of + # dependencies. + # XXX explain better + CouchDBTestCase.setUp(self) + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + TestCaseWithServer.setUp(self) + + def tearDown(self): + CouchDBTestCase.tearDown(self) + TestCaseWithServer.tearDown(self) + + def _test_encrypted_sym_sync(self, passphrase=u'123', doc_size=2, + number_of_docs=1): + """ + Test the complete syncing chain between two soledad dbs using a + Soledad server backed by a couch database. + """ + self.startTwistedServer() + user = 'user-' + uuid4().hex + + # this will store all docs ids to avoid get_all_docs + created_ids = [] + + # instantiate soledad and create a document + sol1 = self._soledad_instance( + user=user, + # token is verified in test_target.make_token_soledad_app + auth_token='auth-token', + passphrase=passphrase) + + # instantiate another soledad using the same secret as the previous + # one (so we can correctly verify the mac of the synced document) + sol2 = self._soledad_instance( + user=user, + prefix='x', + auth_token='auth-token', + secrets_path=sol1._secrets_path, + passphrase=passphrase) + + # ensure remote db exists before syncing + db = CouchDatabase.open_database( + urljoin(self.couch_url, 'user-' + user), + create=True, + ensure_ddocs=True) + + def _db1AssertEmptyDocList(results): + _, doclist = results + self.assertEqual([], doclist) + + def _db1CreateDocs(results): + deferreds = [] + for i in xrange(number_of_docs): + content = binascii.hexlify(os.urandom(doc_size / 2)) + d = sol1.create_doc({'data': content}) + d.addCallback(created_ids.append) + deferreds.append(d) + return defer.DeferredList(deferreds) + + def _db1AssertDocsSyncedToServer(results): + self.assertEqual(number_of_docs, len(created_ids)) + for soldoc in created_ids: + couchdoc = db.get_doc(soldoc.doc_id) + self.assertTrue(couchdoc) + # assert document structure in couch server + self.assertEqual(soldoc.doc_id, couchdoc.doc_id) + self.assertEqual(soldoc.rev, couchdoc.rev) + couch_content = couchdoc.content.keys() + self.assertEqual(6, len(couch_content)) + self.assertTrue(crypto.ENC_JSON_KEY in couch_content) + self.assertTrue(crypto.ENC_SCHEME_KEY in couch_content) + self.assertTrue(crypto.ENC_METHOD_KEY in couch_content) + self.assertTrue(crypto.ENC_IV_KEY in couch_content) + self.assertTrue(crypto.MAC_KEY in couch_content) + self.assertTrue(crypto.MAC_METHOD_KEY in couch_content) + + d = sol1.get_all_docs() + d.addCallback(_db1AssertEmptyDocList) + d.addCallback(_db1CreateDocs) + d.addCallback(lambda _: sol1.sync()) + d.addCallback(_db1AssertDocsSyncedToServer) + + def _db2AssertEmptyDocList(results): + _, doclist = results + self.assertEqual([], doclist) + + def _getAllDocsFromBothDbs(results): + d1 = sol1.get_all_docs() + d2 = sol2.get_all_docs() + return defer.DeferredList([d1, d2]) + + d.addCallback(lambda _: sol2.get_all_docs()) + d.addCallback(_db2AssertEmptyDocList) + d.addCallback(lambda _: sol2.sync()) + d.addCallback(_getAllDocsFromBothDbs) + + def _assertDocSyncedFromDb1ToDb2(results): + r1, r2 = results + _, (gen1, doclist1) = r1 + _, (gen2, doclist2) = r2 + self.assertEqual(number_of_docs, gen1) + self.assertEqual(number_of_docs, gen2) + self.assertEqual(number_of_docs, len(doclist1)) + self.assertEqual(number_of_docs, len(doclist2)) + self.assertEqual(doclist1[0], doclist2[0]) + + d.addCallback(_assertDocSyncedFromDb1ToDb2) + + def _cleanUp(results): + db.delete_database() + db.close() + sol1.close() + sol2.close() + + d.addCallback(_cleanUp) + + return d + + def test_encrypted_sym_sync(self): + return self._test_encrypted_sym_sync() + + def test_encrypted_sym_sync_with_unicode_passphrase(self): + """ + Test the complete syncing chain between two soledad dbs using a + Soledad server backed by a couch database, using an unicode + passphrase. + """ + return self._test_encrypted_sym_sync(passphrase=u'ãáàäéàëíìïóòöõúùüñç') + + def test_sync_very_large_files(self): + """ + Test if Soledad can sync very large files. + """ + self.skipTest( + "Work in progress. For reference, see: " + "https://leap.se/code/issues/7370") + length = 100 * (10 ** 6) # 100 MB + return self._test_encrypted_sym_sync(doc_size=length, number_of_docs=1) + + def test_sync_many_small_files(self): + """ + Test if Soledad can sync many smallfiles. + """ + return self._test_encrypted_sym_sync(doc_size=2, number_of_docs=100) + + +class ConfigurationParsingTest(unittest.TestCase): + + def setUp(self): + self.maxDiff = None + + def test_use_defaults_on_failure(self): + config = load_configuration('this file will never exist') + expected = CONFIG_DEFAULTS + self.assertEquals(expected, config) + + def test_security_values_configuration(self): + # given + config_path = resource_filename('test_soledad', + 'fixture_soledad.conf') + # when + config = load_configuration(config_path) + + # then + expected = {'members': ['user1', 'user2'], + 'members_roles': ['role1', 'role2'], + 'admins': ['user3', 'user4'], + 'admins_roles': ['role3', 'role3']} + self.assertDictEqual(expected, config['database-security']) + + def test_server_values_configuration(self): + # given + config_path = resource_filename('test_soledad', + 'fixture_soledad.conf') + # when + config = load_configuration(config_path) + + # then + expected = {'couch_url': + 'http://soledad:passwd@localhost:5984', + 'create_cmd': + 'sudo -u soledad-admin /usr/bin/create-user-db', + 'admin_netrc': + '/etc/couchdb/couchdb-soledad-admin.netrc', + 'batching': False} + self.assertDictEqual(expected, config['soledad-server']) diff --git a/testing/tests/sync/__init__.py b/testing/tests/sync/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/testing/tests/sync/test_encdecpool.py b/testing/tests/sync/test_encdecpool.py new file mode 100644 index 00000000..82e99a47 --- /dev/null +++ b/testing/tests/sync/test_encdecpool.py @@ -0,0 +1,315 @@ +# -*- coding: utf-8 -*- +# test_encdecpool.py +# Copyright (C) 2015 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Tests for encryption and decryption pool. +""" +import json +from random import shuffle + +from mock import MagicMock +from twisted.internet.defer import inlineCallbacks + +from leap.soledad.client.encdecpool import SyncEncrypterPool +from leap.soledad.client.encdecpool import SyncDecrypterPool + +from leap.soledad.common.document import SoledadDocument +from test_soledad.util import BaseSoledadTest +from twisted.internet import defer +from twisted.test.proto_helpers import MemoryReactorClock + +DOC_ID = "mydoc" +DOC_REV = "rev" +DOC_CONTENT = {'simple': 'document'} + + +class TestSyncEncrypterPool(BaseSoledadTest): + + def setUp(self): + BaseSoledadTest.setUp(self) + crypto = self._soledad._crypto + sync_db = self._soledad._sync_db + self._pool = SyncEncrypterPool(crypto, sync_db) + self._pool.start() + + def tearDown(self): + self._pool.stop() + BaseSoledadTest.tearDown(self) + + @inlineCallbacks + def test_get_encrypted_doc_returns_none(self): + """ + Test that trying to get an encrypted doc from the pool returns None if + the document was never added for encryption. + """ + doc = yield self._pool.get_encrypted_doc(DOC_ID, DOC_REV) + self.assertIsNone(doc) + + @inlineCallbacks + def test_encrypt_doc_and_get_it_back(self): + """ + Test that the pool actually encrypts a document added to the queue. + """ + doc = SoledadDocument( + doc_id=DOC_ID, rev=DOC_REV, json=json.dumps(DOC_CONTENT)) + self._pool.encrypt_doc(doc) + + # exhaustivelly attempt to get the encrypted document + encrypted = None + attempts = 0 + while encrypted is None and attempts < 10: + encrypted = yield self._pool.get_encrypted_doc(DOC_ID, DOC_REV) + attempts += 1 + + self.assertIsNotNone(encrypted) + self.assertTrue(attempts < 10) + + +class TestSyncDecrypterPool(BaseSoledadTest): + + def _insert_doc_cb(self, doc, gen, trans_id): + """ + Method used to mock the sync's return_doc_cb callback. + """ + self._inserted_docs.append((doc, gen, trans_id)) + + def _setup_pool(self, sync_db=None): + sync_db = sync_db or self._soledad._sync_db + return SyncDecrypterPool( + self._soledad._crypto, + sync_db, + source_replica_uid=self._soledad._dbpool.replica_uid, + insert_doc_cb=self._insert_doc_cb) + + def setUp(self): + BaseSoledadTest.setUp(self) + # setup the pool + self._pool = self._setup_pool() + # reset the inserted docs mock + self._inserted_docs = [] + + def tearDown(self): + if self._pool.running: + self._pool.stop() + BaseSoledadTest.tearDown(self) + + def test_insert_received_doc(self): + """ + Test that one document added to the pool is inserted using the + callback. + """ + self._pool.start(1) + self._pool.insert_received_doc( + DOC_ID, DOC_REV, "{}", 1, "trans_id", 1) + + def _assert_doc_was_inserted(_): + self.assertEqual( + self._inserted_docs, + [(SoledadDocument(DOC_ID, DOC_REV, "{}"), 1, u"trans_id")]) + + self._pool.deferred.addCallback(_assert_doc_was_inserted) + return self._pool.deferred + + def test_looping_control(self): + """ + Start and stop cleanly. + """ + self._pool.start(10) + self.assertTrue(self._pool.running) + self._pool.stop() + self.assertFalse(self._pool.running) + self.assertTrue(self._pool.deferred.called) + + def test_sync_id_col_is_created_if_non_existing_in_docs_recvd_table(self): + """ + Test that docs_received table is migrated, and has the sync_id column + """ + mock_run_query = MagicMock(return_value=defer.succeed(None)) + mock_sync_db = MagicMock() + mock_sync_db.runQuery = mock_run_query + pool = self._setup_pool(mock_sync_db) + d = pool.start(10) + pool.stop() + + def assert_trial_to_create_sync_id_column(_): + mock_run_query.assert_called_once_with( + "ALTER TABLE docs_received ADD COLUMN sync_id") + + d.addCallback(assert_trial_to_create_sync_id_column) + return d + + def test_insert_received_doc_many(self): + """ + Test that many documents added to the pool are inserted using the + callback. + """ + many = 100 + self._pool.start(many) + + # insert many docs in the pool + for i in xrange(many): + gen = idx = i + 1 + doc_id = "doc_id: %d" % idx + rev = "rev: %d" % idx + content = {'idx': idx} + trans_id = "trans_id: %d" % idx + self._pool.insert_received_doc( + doc_id, rev, content, gen, trans_id, idx) + + def _assert_doc_was_inserted(_): + self.assertEqual(many, len(self._inserted_docs)) + idx = 1 + for doc, gen, trans_id in self._inserted_docs: + expected_gen = idx + expected_doc_id = "doc_id: %d" % idx + expected_rev = "rev: %d" % idx + expected_content = json.dumps({'idx': idx}) + expected_trans_id = "trans_id: %d" % idx + + self.assertEqual(expected_doc_id, doc.doc_id) + self.assertEqual(expected_rev, doc.rev) + self.assertEqual(expected_content, json.dumps(doc.content)) + self.assertEqual(expected_gen, gen) + self.assertEqual(expected_trans_id, trans_id) + + idx += 1 + + self._pool.deferred.addCallback(_assert_doc_was_inserted) + return self._pool.deferred + + def test_insert_encrypted_received_doc(self): + """ + Test that one encrypted document added to the pool is decrypted and + inserted using the callback. + """ + crypto = self._soledad._crypto + doc = SoledadDocument( + doc_id=DOC_ID, rev=DOC_REV, json=json.dumps(DOC_CONTENT)) + encrypted_content = json.loads(crypto.encrypt_doc(doc)) + + # insert the encrypted document in the pool + self._pool.start(1) + self._pool.insert_encrypted_received_doc( + DOC_ID, DOC_REV, encrypted_content, 1, "trans_id", 1) + + def _assert_doc_was_decrypted_and_inserted(_): + self.assertEqual(1, len(self._inserted_docs)) + self.assertEqual(self._inserted_docs, [(doc, 1, u"trans_id")]) + + self._pool.deferred.addCallback( + _assert_doc_was_decrypted_and_inserted) + return self._pool.deferred + + @inlineCallbacks + def test_processing_order(self): + """ + This test ensures that processing of documents only occur if there is + a sequence in place. + """ + reactor_clock = MemoryReactorClock() + self._pool._loop.clock = reactor_clock + + crypto = self._soledad._crypto + + docs = [] + for i in xrange(1, 10): + i = str(i) + doc = SoledadDocument( + doc_id=DOC_ID + i, rev=DOC_REV + i, + json=json.dumps(DOC_CONTENT)) + encrypted_content = json.loads(crypto.encrypt_doc(doc)) + docs.append((doc, encrypted_content)) + + # insert the encrypted document in the pool + self._pool.start(10) # pool is expecting to process 10 docs + # first three arrives, forming a sequence + for i, (doc, encrypted_content) in enumerate(docs[:3]): + gen = idx = i + 1 + yield self._pool.insert_encrypted_received_doc( + doc.doc_id, doc.rev, encrypted_content, gen, "trans_id", idx) + # last one arrives alone, so it can't be processed + doc, encrypted_content = docs[-1] + yield self._pool.insert_encrypted_received_doc( + doc.doc_id, doc.rev, encrypted_content, 10, "trans_id", 10) + + reactor_clock.advance(self._pool.DECRYPT_LOOP_PERIOD) + yield self._pool._decrypt_and_recurse() + + self.assertEqual(3, self._pool._processed_docs) + + def test_insert_encrypted_received_doc_many(self, many=100): + """ + Test that many encrypted documents added to the pool are decrypted and + inserted using the callback. + """ + crypto = self._soledad._crypto + self._pool.start(many) + docs = [] + + # insert many encrypted docs in the pool + for i in xrange(many): + gen = idx = i + 1 + doc_id = "doc_id: %d" % idx + rev = "rev: %d" % idx + content = {'idx': idx} + trans_id = "trans_id: %d" % idx + + doc = SoledadDocument( + doc_id=doc_id, rev=rev, json=json.dumps(content)) + + encrypted_content = json.loads(crypto.encrypt_doc(doc)) + docs.append((doc_id, rev, encrypted_content, gen, + trans_id, idx)) + shuffle(docs) + + for doc in docs: + self._pool.insert_encrypted_received_doc(*doc) + + def _assert_docs_were_decrypted_and_inserted(_): + self.assertEqual(many, len(self._inserted_docs)) + idx = 1 + for doc, gen, trans_id in self._inserted_docs: + expected_gen = idx + expected_doc_id = "doc_id: %d" % idx + expected_rev = "rev: %d" % idx + expected_content = json.dumps({'idx': idx}) + expected_trans_id = "trans_id: %d" % idx + + self.assertEqual(expected_doc_id, doc.doc_id) + self.assertEqual(expected_rev, doc.rev) + self.assertEqual(expected_content, json.dumps(doc.content)) + self.assertEqual(expected_gen, gen) + self.assertEqual(expected_trans_id, trans_id) + + idx += 1 + + self._pool.deferred.addCallback( + _assert_docs_were_decrypted_and_inserted) + return self._pool.deferred + + @inlineCallbacks + def test_pool_reuse(self): + """ + The pool is reused between syncs, this test verifies that + reusing is fine. + """ + for i in xrange(3): + yield self.test_insert_encrypted_received_doc_many(5) + self._inserted_docs = [] + decrypted_docs = yield self._pool._get_docs(encrypted=False) + # check that decrypted docs staging is clean + self.assertEquals([], decrypted_docs) + self._pool.stop() diff --git a/testing/tests/sync/test_sync.py b/testing/tests/sync/test_sync.py new file mode 100644 index 00000000..095884ce --- /dev/null +++ b/testing/tests/sync/test_sync.py @@ -0,0 +1,216 @@ +# -*- coding: utf-8 -*- +# test_sync.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import json +import tempfile +import threading +import time + +from urlparse import urljoin +from twisted.internet import defer + +from testscenarios import TestWithScenarios + +from leap.soledad.common import couch +from leap.soledad.client import sync + +from test_soledad import u1db_tests as tests +from test_soledad.u1db_tests import TestCaseWithServer +from test_soledad.u1db_tests import simple_doc +from test_soledad.util import make_token_soledad_app +from test_soledad.util import make_soledad_document_for_test +from test_soledad.util import soledad_sync_target +from test_soledad.util import BaseSoledadTest +from test_soledad.util import SoledadWithCouchServerMixin +from test_soledad.util import CouchDBTestCase + + +class InterruptableSyncTestCase( + BaseSoledadTest, CouchDBTestCase, TestCaseWithServer): + + """ + Tests for encrypted sync using Soledad server backed by a couch database. + """ + + @staticmethod + def make_app_with_state(state): + return make_token_soledad_app(state) + + make_document_for_test = make_soledad_document_for_test + + sync_target = soledad_sync_target + + def make_app(self): + self.request_state = couch.CouchServerState(self.couch_url) + return self.make_app_with_state(self.request_state) + + def setUp(self): + TestCaseWithServer.setUp(self) + CouchDBTestCase.setUp(self) + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + + def tearDown(self): + CouchDBTestCase.tearDown(self) + TestCaseWithServer.tearDown(self) + + def test_interruptable_sync(self): + """ + Test if Soledad can sync many smallfiles. + """ + + self.skipTest("Sync is currently not interruptable.") + + class _SyncInterruptor(threading.Thread): + + """ + A thread meant to interrupt the sync process. + """ + + def __init__(self, soledad, couchdb): + self._soledad = soledad + self._couchdb = couchdb + threading.Thread.__init__(self) + + def run(self): + while db._get_generation() < 2: + # print "WAITING %d" % db._get_generation() + time.sleep(0.1) + self._soledad.stop_sync() + time.sleep(1) + + number_of_docs = 10 + self.startServer() + + # instantiate soledad and create a document + sol = self._soledad_instance( + user='user-uuid', server_url=self.getURL()) + + # ensure remote db exists before syncing + db = couch.CouchDatabase.open_database( + urljoin(self.couch_url, 'user-user-uuid'), + create=True, + ensure_ddocs=True) + + # create interruptor thread + t = _SyncInterruptor(sol, db) + t.start() + + d = sol.get_all_docs() + d.addCallback(lambda results: self.assertEqual([], results[1])) + + def _create_docs(results): + # create many small files + deferreds = [] + for i in range(0, number_of_docs): + deferreds.append(sol.create_doc(json.loads(simple_doc))) + return defer.DeferredList(deferreds) + + # sync with server + d.addCallback(_create_docs) + d.addCallback(lambda _: sol.get_all_docs()) + d.addCallback( + lambda results: self.assertEqual(number_of_docs, len(results[1]))) + d.addCallback(lambda _: sol.sync()) + d.addCallback(lambda _: t.join()) + d.addCallback(lambda _: db.get_all_docs()) + d.addCallback( + lambda results: self.assertNotEqual( + number_of_docs, len(results[1]))) + d.addCallback(lambda _: sol.sync()) + d.addCallback(lambda _: db.get_all_docs()) + d.addCallback( + lambda results: self.assertEqual(number_of_docs, len(results[1]))) + + def _tear_down(results): + db.delete_database() + db.close() + sol.close() + + d.addCallback(_tear_down) + return d + + +class TestSoledadDbSync( + TestWithScenarios, + SoledadWithCouchServerMixin, + tests.TestCaseWithServer): + + """ + Test db.sync remote sync shortcut + """ + + scenarios = [ + ('py-token-http', { + 'make_app_with_state': make_token_soledad_app, + 'make_database_for_test': tests.make_memory_database_for_test, + 'token': True + }), + ] + + oauth = False + token = False + + def setUp(self): + """ + Need to explicitely invoke inicialization on all bases. + """ + SoledadWithCouchServerMixin.setUp(self) + self.startTwistedServer() + self.db = self.make_database_for_test(self, 'test1') + self.db2 = self.request_state._create_database(replica_uid='test') + + def tearDown(self): + """ + Need to explicitely invoke destruction on all bases. + """ + SoledadWithCouchServerMixin.tearDown(self) + # tests.TestCaseWithServer.tearDown(self) + + def do_sync(self): + """ + Perform sync using SoledadSynchronizer, SoledadSyncTarget + and Token auth. + """ + target = soledad_sync_target( + self, self.db2._dbname, + source_replica_uid=self._soledad._dbpool.replica_uid) + self.addCleanup(target.close) + return sync.SoledadSynchronizer( + self.db, + target).sync(defer_decryption=False) + + @defer.inlineCallbacks + def test_db_sync(self): + """ + Test sync. + + Adapted to check for encrypted content. + """ + + doc1 = self.db.create_doc_from_json(tests.simple_doc) + doc2 = self.db2.create_doc_from_json(tests.nested_doc) + + local_gen_before_sync = yield self.do_sync() + gen, _, changes = self.db.whats_changed(local_gen_before_sync) + self.assertEqual(1, len(changes)) + self.assertEqual(doc2.doc_id, changes[0][0]) + self.assertEqual(1, gen - local_gen_before_sync) + self.assertGetEncryptedDoc( + self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) + self.assertGetEncryptedDoc( + self.db, doc2.doc_id, doc2.rev, tests.nested_doc, False) + + # TODO: add u1db.tests.test_sync.TestRemoteSyncIntegration diff --git a/testing/tests/sync/test_sync_deferred.py b/testing/tests/sync/test_sync_deferred.py new file mode 100644 index 00000000..4948aaf8 --- /dev/null +++ b/testing/tests/sync/test_sync_deferred.py @@ -0,0 +1,196 @@ +# test_sync_deferred.py +# Copyright (C) 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test Leap backend bits: sync with deferred encryption/decryption. +""" +import time +import os +import random +import string +import shutil + +from urlparse import urljoin + +from twisted.internet import defer + +from leap.soledad.common import couch + +from leap.soledad.client import sync +from leap.soledad.client.sqlcipher import SQLCipherOptions +from leap.soledad.client.sqlcipher import SQLCipherDatabase + +from testscenarios import TestWithScenarios + +from test_soledad import u1db_tests as tests +from test_soledad.util import ADDRESS +from test_soledad.util import SoledadWithCouchServerMixin +from test_soledad.util import make_soledad_app +from test_soledad.util import soledad_sync_target + + +# Just to make clear how this test is different... :) +DEFER_DECRYPTION = True + +WAIT_STEP = 1 +MAX_WAIT = 10 +DBPASS = "pass" + + +class BaseSoledadDeferredEncTest(SoledadWithCouchServerMixin): + + """ + Another base class for testing the deferred encryption/decryption during + the syncs, using the intermediate database. + """ + defer_sync_encryption = True + + def setUp(self): + SoledadWithCouchServerMixin.setUp(self) + self.startTwistedServer() + # config info + self.db1_file = os.path.join(self.tempdir, "db1.u1db") + os.unlink(self.db1_file) + self.db_pass = DBPASS + self.email = ADDRESS + + # get a random prefix for each test, so we do not mess with + # concurrency during initialization and shutting down of + # each local db. + self.rand_prefix = ''.join( + map(lambda x: random.choice(string.ascii_letters), range(6))) + + # open test dbs: db1 will be the local sqlcipher db (which + # instantiates a syncdb). We use the self._soledad instance that was + # already created on some setUp method. + import binascii + tohex = binascii.b2a_hex + key = tohex(self._soledad.secrets.get_local_storage_key()) + sync_db_key = tohex(self._soledad.secrets.get_sync_db_key()) + dbpath = self._soledad._local_db_path + + self.opts = SQLCipherOptions( + dbpath, key, is_raw_key=True, create=False, + defer_encryption=True, sync_db_key=sync_db_key) + self.db1 = SQLCipherDatabase(self.opts) + + self.db2 = self.request_state._create_database('test') + + def tearDown(self): + # XXX should not access "private" attrs + shutil.rmtree(os.path.dirname(self._soledad._local_db_path)) + SoledadWithCouchServerMixin.tearDown(self) + + +class SyncTimeoutError(Exception): + + """ + Dummy exception to notify timeout during sync. + """ + pass + + +class TestSoledadDbSyncDeferredEncDecr( + TestWithScenarios, + BaseSoledadDeferredEncTest, + tests.TestCaseWithServer): + + """ + Test db.sync remote sync shortcut. + Case with deferred encryption and decryption: using the intermediate + syncdb. + """ + + scenarios = [ + ('http', { + 'make_app_with_state': make_soledad_app, + 'make_database_for_test': tests.make_memory_database_for_test, + }), + ] + + oauth = False + token = True + + def setUp(self): + """ + Need to explicitely invoke inicialization on all bases. + """ + BaseSoledadDeferredEncTest.setUp(self) + self.server = self.server_thread = None + self.syncer = None + + def tearDown(self): + """ + Need to explicitely invoke destruction on all bases. + """ + dbsyncer = getattr(self, 'dbsyncer', None) + if dbsyncer: + dbsyncer.close() + BaseSoledadDeferredEncTest.tearDown(self) + + def do_sync(self): + """ + Perform sync using SoledadSynchronizer, SoledadSyncTarget + and Token auth. + """ + replica_uid = self._soledad._dbpool.replica_uid + sync_db = self._soledad._sync_db + sync_enc_pool = self._soledad._sync_enc_pool + dbsyncer = self._soledad._dbsyncer # Soledad.sync uses the dbsyncer + + target = soledad_sync_target( + self, self.db2._dbname, + source_replica_uid=replica_uid, + sync_db=sync_db, + sync_enc_pool=sync_enc_pool) + self.addCleanup(target.close) + return sync.SoledadSynchronizer( + dbsyncer, + target).sync(defer_decryption=True) + + def wait_for_sync(self): + """ + Wait for sync to finish. + """ + wait = 0 + syncer = self.syncer + if syncer is not None: + while syncer.syncing: + time.sleep(WAIT_STEP) + wait += WAIT_STEP + if wait >= MAX_WAIT: + raise SyncTimeoutError + + @defer.inlineCallbacks + def test_db_sync(self): + """ + Test sync. + + Adapted to check for encrypted content. + """ + doc1 = self.db1.create_doc_from_json(tests.simple_doc) + doc2 = self.db2.create_doc_from_json(tests.nested_doc) + local_gen_before_sync = yield self.do_sync() + + gen, _, changes = self.db1.whats_changed(local_gen_before_sync) + self.assertEqual(1, len(changes)) + + self.assertEqual(doc2.doc_id, changes[0][0]) + self.assertEqual(1, gen - local_gen_before_sync) + + self.assertGetEncryptedDoc( + self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) + self.assertGetEncryptedDoc( + self.db1, doc2.doc_id, doc2.rev, tests.nested_doc, False) diff --git a/testing/tests/sync/test_sync_mutex.py b/testing/tests/sync/test_sync_mutex.py new file mode 100644 index 00000000..787cfee8 --- /dev/null +++ b/testing/tests/sync/test_sync_mutex.py @@ -0,0 +1,135 @@ +# -*- coding: utf-8 -*- +# test_sync_mutex.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +Test that synchronization is a critical section and, as such, there might not +be two concurrent synchronization processes at the same time. +""" + + +import time +import uuid +import tempfile +import shutil + +from urlparse import urljoin + +from twisted.internet import defer + +from leap.soledad.client.sync import SoledadSynchronizer + +from leap.soledad.common.couch.state import CouchServerState +from leap.soledad.common.couch import CouchDatabase +from test_soledad.u1db_tests import TestCaseWithServer + +from test_soledad.util import CouchDBTestCase +from test_soledad.util import BaseSoledadTest +from test_soledad.util import make_token_soledad_app +from test_soledad.util import make_soledad_document_for_test +from test_soledad.util import soledad_sync_target + + +# monkey-patch the soledad synchronizer so it stores start and finish times + +_old_sync = SoledadSynchronizer.sync + + +def _timed_sync(self, defer_decryption=True): + t = time.time() + + sync_id = uuid.uuid4() + + if not getattr(self.source, 'sync_times', False): + self.source.sync_times = {} + + self.source.sync_times[sync_id] = {'start': t} + + def _store_finish_time(passthrough): + t = time.time() + self.source.sync_times[sync_id]['end'] = t + return passthrough + + d = _old_sync(self, defer_decryption=defer_decryption) + d.addBoth(_store_finish_time) + return d + +SoledadSynchronizer.sync = _timed_sync + +# -- end of monkey-patching + + +class TestSyncMutex( + BaseSoledadTest, CouchDBTestCase, TestCaseWithServer): + + @staticmethod + def make_app_with_state(state): + return make_token_soledad_app(state) + + make_document_for_test = make_soledad_document_for_test + + sync_target = soledad_sync_target + + def make_app(self): + self.request_state = CouchServerState(self.couch_url) + return self.make_app_with_state(self.request_state) + + def setUp(self): + TestCaseWithServer.setUp(self) + CouchDBTestCase.setUp(self) + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + self.user = ('user-%s' % uuid.uuid4().hex) + + def tearDown(self): + CouchDBTestCase.tearDown(self) + TestCaseWithServer.tearDown(self) + shutil.rmtree(self.tempdir) + + def test_two_concurrent_syncs_do_not_overlap_no_docs(self): + self.startServer() + + # ensure remote db exists before syncing + db = CouchDatabase.open_database( + urljoin(self.couch_url, 'user-' + self.user), + create=True, + ensure_ddocs=True) + + sol = self._soledad_instance( + user=self.user, server_url=self.getURL()) + + d1 = sol.sync() + d2 = sol.sync() + + def _assert_syncs_do_not_overlap(thearg): + # recover sync times + sync_times = [] + for key in sol._dbsyncer.sync_times: + sync_times.append(sol._dbsyncer.sync_times[key]) + sync_times.sort(key=lambda s: s['start']) + + self.assertTrue( + (sync_times[0]['start'] < sync_times[0]['end'] and + sync_times[0]['end'] < sync_times[1]['start'] and + sync_times[1]['start'] < sync_times[1]['end'])) + + db.delete_database() + db.close() + sol.close() + + d = defer.gatherResults([d1, d2]) + d.addBoth(_assert_syncs_do_not_overlap) + return d diff --git a/testing/tests/sync/test_sync_target.py b/testing/tests/sync/test_sync_target.py new file mode 100644 index 00000000..964468ce --- /dev/null +++ b/testing/tests/sync/test_sync_target.py @@ -0,0 +1,968 @@ +# -*- coding: utf-8 -*- +# test_sync_target.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test Leap backend bits: sync target +""" +import cStringIO +import os +import time +import json +import random +import string +import shutil +from uuid import uuid4 + +from testscenarios import TestWithScenarios +from twisted.internet import defer + +from leap.soledad.client import http_target as target +from leap.soledad.client import crypto +from leap.soledad.client.sqlcipher import SQLCipherU1DBSync +from leap.soledad.client.sqlcipher import SQLCipherOptions +from leap.soledad.client.sqlcipher import SQLCipherDatabase + +from leap.soledad.common import l2db + +from leap.soledad.common.document import SoledadDocument +from test_soledad import u1db_tests as tests +from test_soledad.util import make_sqlcipher_database_for_test +from test_soledad.util import make_soledad_app +from test_soledad.util import make_token_soledad_app +from test_soledad.util import make_soledad_document_for_test +from test_soledad.util import soledad_sync_target +from test_soledad.util import SoledadWithCouchServerMixin +from test_soledad.util import ADDRESS +from test_soledad.util import SQLCIPHER_SCENARIOS + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_remote_sync_target`. +# ----------------------------------------------------------------------------- + +class TestSoledadParseReceivedDocResponse(SoledadWithCouchServerMixin): + + """ + Some tests had to be copied to this class so we can instantiate our own + target. + """ + + def setUp(self): + SoledadWithCouchServerMixin.setUp(self) + creds = {'token': { + 'uuid': 'user-uuid', + 'token': 'auth-token', + }} + self.target = target.SoledadHTTPSyncTarget( + self.couch_url, + uuid4().hex, + creds, + self._soledad._crypto, + None) + + def tearDown(self): + self.target.close() + SoledadWithCouchServerMixin.tearDown(self) + + def test_extra_comma(self): + """ + Test adapted to use encrypted content. + """ + doc = SoledadDocument('i', rev='r') + doc.content = {} + _crypto = self._soledad._crypto + key = _crypto.doc_passphrase(doc.doc_id) + secret = _crypto.secret + + enc_json = crypto.encrypt_docstr( + doc.get_json(), doc.doc_id, doc.rev, + key, secret) + + with self.assertRaises(l2db.errors.BrokenSyncStream): + self.target._parse_received_doc_response("[\r\n{},\r\n]") + + with self.assertRaises(l2db.errors.BrokenSyncStream): + self.target._parse_received_doc_response( + ('[\r\n{},\r\n{"id": "i", "rev": "r", ' + + '"content": %s, "gen": 3, "trans_id": "T-sid"}' + + ',\r\n]') % json.dumps(enc_json)) + + def test_wrong_start(self): + with self.assertRaises(l2db.errors.BrokenSyncStream): + self.target._parse_received_doc_response("{}\r\n]") + + with self.assertRaises(l2db.errors.BrokenSyncStream): + self.target._parse_received_doc_response("\r\n{}\r\n]") + + with self.assertRaises(l2db.errors.BrokenSyncStream): + self.target._parse_received_doc_response("") + + def test_wrong_end(self): + with self.assertRaises(l2db.errors.BrokenSyncStream): + self.target._parse_received_doc_response("[\r\n{}") + + with self.assertRaises(l2db.errors.BrokenSyncStream): + self.target._parse_received_doc_response("[\r\n") + + def test_missing_comma(self): + with self.assertRaises(l2db.errors.BrokenSyncStream): + self.target._parse_received_doc_response( + '[\r\n{}\r\n{"id": "i", "rev": "r", ' + '"content": "c", "gen": 3}\r\n]') + + def test_no_entries(self): + with self.assertRaises(l2db.errors.BrokenSyncStream): + self.target._parse_received_doc_response("[\r\n]") + + def test_error_in_stream(self): + with self.assertRaises(l2db.errors.BrokenSyncStream): + self.target._parse_received_doc_response( + '[\r\n{"new_generation": 0},' + '\r\n{"error": "unavailable"}\r\n') + + with self.assertRaises(l2db.errors.BrokenSyncStream): + self.target._parse_received_doc_response( + '[\r\n{"error": "unavailable"}\r\n') + + with self.assertRaises(l2db.errors.BrokenSyncStream): + self.target._parse_received_doc_response('[\r\n{"error": "?"}\r\n') + +# +# functions for TestRemoteSyncTargets +# + + +def make_local_db_and_soledad_target( + test, path='test', + source_replica_uid=uuid4().hex): + test.startTwistedServer() + replica_uid = os.path.basename(path) + db = test.request_state._create_database(replica_uid) + sync_db = test._soledad._sync_db + sync_enc_pool = test._soledad._sync_enc_pool + st = soledad_sync_target( + test, db._dbname, + source_replica_uid=source_replica_uid, + sync_db=sync_db, + sync_enc_pool=sync_enc_pool) + return db, st + + +def make_local_db_and_token_soledad_target( + test, + source_replica_uid=uuid4().hex): + db, st = make_local_db_and_soledad_target( + test, path='test', + source_replica_uid=source_replica_uid) + st.set_token_credentials('user-uuid', 'auth-token') + return db, st + + +class TestSoledadSyncTarget( + TestWithScenarios, + SoledadWithCouchServerMixin, + tests.TestCaseWithServer): + + scenarios = [ + ('token_soledad', + {'make_app_with_state': make_token_soledad_app, + 'make_document_for_test': make_soledad_document_for_test, + 'create_db_and_target': make_local_db_and_token_soledad_target, + 'make_database_for_test': make_sqlcipher_database_for_test, + 'sync_target': soledad_sync_target}), + ] + + def getSyncTarget(self, path=None, source_replica_uid=uuid4().hex): + if self.port is None: + self.startTwistedServer() + sync_db = self._soledad._sync_db + sync_enc_pool = self._soledad._sync_enc_pool + if path is None: + path = self.db2._dbname + target = self.sync_target( + self, path, + source_replica_uid=source_replica_uid, + sync_db=sync_db, + sync_enc_pool=sync_enc_pool) + self.addCleanup(target.close) + return target + + def setUp(self): + TestWithScenarios.setUp(self) + SoledadWithCouchServerMixin.setUp(self) + self.startTwistedServer() + self.db1 = make_sqlcipher_database_for_test(self, 'test1') + self.db2 = self.request_state._create_database('test') + + def tearDown(self): + # db2, _ = self.request_state.ensure_database('test2') + self.delete_db(self.db2._dbname) + self.db1.close() + SoledadWithCouchServerMixin.tearDown(self) + TestWithScenarios.tearDown(self) + + @defer.inlineCallbacks + def test_sync_exchange_send(self): + """ + Test for sync exchanging send of document. + + This test was adapted to decrypt remote content before assert. + """ + db = self.db2 + remote_target = self.getSyncTarget() + other_docs = [] + + def receive_doc(doc, gen, trans_id): + other_docs.append((doc.doc_id, doc.rev, doc.get_json())) + + doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + new_gen, trans_id = yield remote_target.sync_exchange( + [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=receive_doc, + defer_decryption=False) + self.assertEqual(1, new_gen) + self.assertGetEncryptedDoc( + db, 'doc-here', 'replica:1', '{"value": "here"}', False) + + @defer.inlineCallbacks + def test_sync_exchange_send_failure_and_retry_scenario(self): + """ + Test for sync exchange failure and retry. + + This test was adapted to decrypt remote content before assert. + """ + + def blackhole_getstderr(inst): + return cStringIO.StringIO() + + db = self.db2 + _put_doc_if_newer = db._put_doc_if_newer + trigger_ids = ['doc-here2'] + + def bomb_put_doc_if_newer(self, doc, save_conflict, + replica_uid=None, replica_gen=None, + replica_trans_id=None, number_of_docs=None, + doc_idx=None, sync_id=None): + if doc.doc_id in trigger_ids: + raise l2db.errors.U1DBError + return _put_doc_if_newer(doc, save_conflict=save_conflict, + replica_uid=replica_uid, + replica_gen=replica_gen, + replica_trans_id=replica_trans_id, + number_of_docs=number_of_docs, + doc_idx=doc_idx, sync_id=sync_id) + from leap.soledad.common.backend import SoledadBackend + self.patch( + SoledadBackend, '_put_doc_if_newer', bomb_put_doc_if_newer) + remote_target = self.getSyncTarget( + source_replica_uid='replica') + other_changes = [] + + def receive_doc(doc, gen, trans_id): + other_changes.append( + (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) + + doc1 = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + doc2 = self.make_document('doc-here2', 'replica:1', + '{"value": "here2"}') + + with self.assertRaises(l2db.errors.U1DBError): + yield remote_target.sync_exchange( + [(doc1, 10, 'T-sid'), (doc2, 11, 'T-sud')], + 'replica', + last_known_generation=0, + last_known_trans_id=None, + insert_doc_cb=receive_doc, + defer_decryption=False) + + self.assertGetEncryptedDoc( + db, 'doc-here', 'replica:1', '{"value": "here"}', + False) + self.assertEqual( + (10, 'T-sid'), db._get_replica_gen_and_trans_id('replica')) + self.assertEqual([], other_changes) + # retry + trigger_ids = [] + new_gen, trans_id = yield remote_target.sync_exchange( + [(doc2, 11, 'T-sud')], 'replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=receive_doc, + defer_decryption=False) + self.assertGetEncryptedDoc( + db, 'doc-here2', 'replica:1', '{"value": "here2"}', + False) + self.assertEqual( + (11, 'T-sud'), db._get_replica_gen_and_trans_id('replica')) + self.assertEqual(2, new_gen) + self.assertEqual( + ('doc-here', 'replica:1', '{"value": "here"}', 1), + other_changes[0][:-1]) + + @defer.inlineCallbacks + def test_sync_exchange_send_ensure_callback(self): + """ + Test for sync exchange failure and retry. + + This test was adapted to decrypt remote content before assert. + """ + remote_target = self.getSyncTarget() + other_docs = [] + replica_uid_box = [] + + def receive_doc(doc, gen, trans_id): + other_docs.append((doc.doc_id, doc.rev, doc.get_json())) + + def ensure_cb(replica_uid): + replica_uid_box.append(replica_uid) + + doc = self.make_document('doc-here', 'replica:1', '{"value": "here"}') + new_gen, trans_id = yield remote_target.sync_exchange( + [(doc, 10, 'T-sid')], 'replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=receive_doc, + ensure_callback=ensure_cb, defer_decryption=False) + self.assertEqual(1, new_gen) + db = self.db2 + self.assertEqual(1, len(replica_uid_box)) + self.assertEqual(db._replica_uid, replica_uid_box[0]) + self.assertGetEncryptedDoc( + db, 'doc-here', 'replica:1', '{"value": "here"}', False) + + def test_sync_exchange_in_stream_error(self): + self.skipTest("bypass this test because our sync_exchange process " + "does not return u1db error 503 \"unavailable\" for " + "now") + + @defer.inlineCallbacks + def test_get_sync_info(self): + db = self.db2 + db._set_replica_gen_and_trans_id('other-id', 1, 'T-transid') + remote_target = self.getSyncTarget( + source_replica_uid='other-id') + sync_info = yield remote_target.get_sync_info('other-id') + self.assertEqual( + ('test', 0, '', 1, 'T-transid'), + sync_info) + + @defer.inlineCallbacks + def test_record_sync_info(self): + remote_target = self.getSyncTarget( + source_replica_uid='other-id') + yield remote_target.record_sync_info('other-id', 2, 'T-transid') + self.assertEqual((2, 'T-transid'), + self.db2._get_replica_gen_and_trans_id('other-id')) + + @defer.inlineCallbacks + def test_sync_exchange_receive(self): + db = self.db2 + doc = db.create_doc_from_json('{"value": "there"}') + remote_target = self.getSyncTarget() + other_changes = [] + + def receive_doc(doc, gen, trans_id): + other_changes.append( + (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) + + new_gen, trans_id = yield remote_target.sync_exchange( + [], 'replica', last_known_generation=0, last_known_trans_id=None, + insert_doc_cb=receive_doc) + self.assertEqual(1, new_gen) + self.assertEqual( + (doc.doc_id, doc.rev, '{"value": "there"}', 1), + other_changes[0][:-1]) + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_sync`. +# ----------------------------------------------------------------------------- + +target_scenarios = [ + ('mem,token_soledad', + {'create_db_and_target': make_local_db_and_token_soledad_target, + 'make_app_with_state': make_soledad_app, + 'make_database_for_test': tests.make_memory_database_for_test, + 'copy_database_for_test': tests.copy_memory_database_for_test, + 'make_document_for_test': tests.make_document_for_test}) +] + + +class SoledadDatabaseSyncTargetTests( + TestWithScenarios, + SoledadWithCouchServerMixin, + tests.DatabaseBaseTests, + tests.TestCaseWithServer): + """ + Adaptation of u1db.tests.test_sync.DatabaseSyncTargetTests. + """ + + # TODO: implement _set_trace_hook(_shallow) in SoledadHTTPSyncTarget so + # skipped tests can be succesfully executed. + + scenarios = target_scenarios + + whitebox = False + + def setUp(self): + tests.TestCaseWithServer.setUp(self) + self.other_changes = [] + SoledadWithCouchServerMixin.setUp(self) + self.db, self.st = make_local_db_and_soledad_target(self) + + def tearDown(self): + self.db.close() + self.st.close() + tests.TestCaseWithServer.tearDown(self) + SoledadWithCouchServerMixin.tearDown(self) + + def set_trace_hook(self, callback, shallow=False): + setter = (self.st._set_trace_hook if not shallow else + self.st._set_trace_hook_shallow) + try: + setter(callback) + except NotImplementedError: + self.skipTest("%s does not implement _set_trace_hook" + % (self.st.__class__.__name__,)) + + @defer.inlineCallbacks + def test_sync_exchange(self): + """ + Test sync exchange. + + This test was adapted to decrypt remote content before assert. + """ + docs_by_gen = [ + (self.make_document('doc-id', 'replica:1', tests.simple_doc), 10, + 'T-sid')] + new_gen, trans_id = yield self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=self.receive_doc, + defer_decryption=False) + self.assertGetEncryptedDoc( + self.db, 'doc-id', 'replica:1', tests.simple_doc, False) + self.assertTransactionLog(['doc-id'], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 1, last_trans_id), + (self.other_changes, new_gen, last_trans_id)) + sync_info = yield self.st.get_sync_info('replica') + self.assertEqual(10, sync_info[3]) + + @defer.inlineCallbacks + def test_sync_exchange_push_many(self): + """ + Test sync exchange. + + This test was adapted to decrypt remote content before assert. + """ + docs_by_gen = [ + (self.make_document( + 'doc-id', 'replica:1', tests.simple_doc), 10, 'T-1'), + (self.make_document( + 'doc-id2', 'replica:1', tests.nested_doc), 11, 'T-2')] + new_gen, trans_id = yield self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=self.receive_doc, + defer_decryption=False) + self.assertGetEncryptedDoc( + self.db, 'doc-id', 'replica:1', tests.simple_doc, False) + self.assertGetEncryptedDoc( + self.db, 'doc-id2', 'replica:1', tests.nested_doc, False) + self.assertTransactionLog(['doc-id', 'doc-id2'], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 2, last_trans_id), + (self.other_changes, new_gen, trans_id)) + sync_info = yield self.st.get_sync_info('replica') + self.assertEqual(11, sync_info[3]) + + @defer.inlineCallbacks + def test_sync_exchange_returns_many_new_docs(self): + """ + Test sync exchange. + + This test was adapted to avoid JSON serialization comparison as local + and remote representations might differ. It looks directly at the + doc's contents instead. + """ + doc = self.db.create_doc_from_json(tests.simple_doc) + doc2 = self.db.create_doc_from_json(tests.nested_doc) + self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) + new_gen, _ = yield self.st.sync_exchange( + [], 'other-replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=self.receive_doc, + defer_decryption=False) + self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) + self.assertEqual(2, new_gen) + self.assertEqual( + [(doc.doc_id, doc.rev, 1), + (doc2.doc_id, doc2.rev, 2)], + [c[:-3] + c[-2:-1] for c in self.other_changes]) + self.assertEqual( + json.loads(tests.simple_doc), + json.loads(self.other_changes[0][2])) + self.assertEqual( + json.loads(tests.nested_doc), + json.loads(self.other_changes[1][2])) + if self.whitebox: + self.assertEqual( + self.db._last_exchange_log['return'], + {'last_gen': 2, 'docs': + [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]}) + + def receive_doc(self, doc, gen, trans_id): + self.other_changes.append( + (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) + + def test_get_sync_target(self): + self.assertIsNot(None, self.st) + + @defer.inlineCallbacks + def test_get_sync_info(self): + sync_info = yield self.st.get_sync_info('other') + self.assertEqual( + ('test', 0, '', 0, ''), sync_info) + + @defer.inlineCallbacks + def test_create_doc_updates_sync_info(self): + sync_info = yield self.st.get_sync_info('other') + self.assertEqual( + ('test', 0, '', 0, ''), sync_info) + self.db.create_doc_from_json(tests.simple_doc) + sync_info = yield self.st.get_sync_info('other') + self.assertEqual(1, sync_info[1]) + + @defer.inlineCallbacks + def test_record_sync_info(self): + yield self.st.record_sync_info('replica', 10, 'T-transid') + sync_info = yield self.st.get_sync_info('replica') + self.assertEqual( + ('test', 0, '', 10, 'T-transid'), sync_info) + + @defer.inlineCallbacks + def test_sync_exchange_deleted(self): + doc = self.db.create_doc_from_json('{}') + edit_rev = 'replica:1|' + doc.rev + docs_by_gen = [ + (self.make_document(doc.doc_id, edit_rev, None), 10, 'T-sid')] + new_gen, trans_id = yield self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=self.receive_doc) + self.assertGetDocIncludeDeleted( + self.db, doc.doc_id, edit_rev, None, False) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 2, last_trans_id), + (self.other_changes, new_gen, trans_id)) + sync_info = yield self.st.get_sync_info('replica') + self.assertEqual(10, sync_info[3]) + + @defer.inlineCallbacks + def test_sync_exchange_refuses_conflicts(self): + doc = self.db.create_doc_from_json(tests.simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + (self.make_document(doc.doc_id, 'replica:1', new_doc), 10, + 'T-sid')] + new_gen, _ = yield self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertEqual( + (doc.doc_id, doc.rev, tests.simple_doc, 1), + self.other_changes[0][:-1]) + self.assertEqual(1, new_gen) + if self.whitebox: + self.assertEqual(self.db._last_exchange_log['return'], + {'last_gen': 1, 'docs': [(doc.doc_id, doc.rev)]}) + + @defer.inlineCallbacks + def test_sync_exchange_ignores_convergence(self): + doc = self.db.create_doc_from_json(tests.simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + gen, txid = self.db._get_generation_info() + docs_by_gen = [ + (self.make_document(doc.doc_id, doc.rev, tests.simple_doc), + 10, 'T-sid')] + new_gen, _ = yield self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=gen, + last_known_trans_id=txid, insert_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertEqual(([], 1), (self.other_changes, new_gen)) + + @defer.inlineCallbacks + def test_sync_exchange_returns_new_docs(self): + doc = self.db.create_doc_from_json(tests.simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_gen, _ = yield self.st.sync_exchange( + [], 'other-replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertEqual( + (doc.doc_id, doc.rev, tests.simple_doc, 1), + self.other_changes[0][:-1]) + self.assertEqual(1, new_gen) + if self.whitebox: + self.assertEqual(self.db._last_exchange_log['return'], + {'last_gen': 1, 'docs': [(doc.doc_id, doc.rev)]}) + + @defer.inlineCallbacks + def test_sync_exchange_returns_deleted_docs(self): + doc = self.db.create_doc_from_json(tests.simple_doc) + self.db.delete_doc(doc) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + new_gen, _ = yield self.st.sync_exchange( + [], 'other-replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + self.assertEqual( + (doc.doc_id, doc.rev, None, 2), self.other_changes[0][:-1]) + self.assertEqual(2, new_gen) + if self.whitebox: + self.assertEqual(self.db._last_exchange_log['return'], + {'last_gen': 2, 'docs': [(doc.doc_id, doc.rev)]}) + + @defer.inlineCallbacks + def test_sync_exchange_getting_newer_docs(self): + doc = self.db.create_doc_from_json(tests.simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + 'T-sid')] + new_gen, _ = yield self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + self.assertEqual(([], 2), (self.other_changes, new_gen)) + + @defer.inlineCallbacks + def test_sync_exchange_with_concurrent_updates_of_synced_doc(self): + expected = [] + + def before_whatschanged_cb(state): + if state != 'before whats_changed': + return + cont = '{"key": "cuncurrent"}' + conc_rev = self.db.put_doc( + self.make_document(doc.doc_id, 'test:1|z:2', cont)) + expected.append((doc.doc_id, conc_rev, cont, 3)) + + self.set_trace_hook(before_whatschanged_cb) + doc = self.db.create_doc_from_json(tests.simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + 'T-sid')] + new_gen, _ = yield self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=self.receive_doc) + self.assertEqual(expected, [c[:-1] for c in self.other_changes]) + self.assertEqual(3, new_gen) + + @defer.inlineCallbacks + def test_sync_exchange_with_concurrent_updates(self): + + def after_whatschanged_cb(state): + if state != 'after whats_changed': + return + self.db.create_doc_from_json('{"new": "doc"}') + + self.set_trace_hook(after_whatschanged_cb) + doc = self.db.create_doc_from_json(tests.simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + 'T-sid')] + new_gen, _ = yield self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=self.receive_doc) + self.assertEqual(([], 2), (self.other_changes, new_gen)) + + @defer.inlineCallbacks + def test_sync_exchange_converged_handling(self): + doc = self.db.create_doc_from_json(tests.simple_doc) + docs_by_gen = [ + (self.make_document('new', 'other:1', '{}'), 4, 'T-foo'), + (self.make_document(doc.doc_id, doc.rev, doc.get_json()), 5, + 'T-bar')] + new_gen, _ = yield self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, insert_doc_cb=self.receive_doc) + self.assertEqual(([], 2), (self.other_changes, new_gen)) + + @defer.inlineCallbacks + def test_sync_exchange_detect_incomplete_exchange(self): + def before_get_docs_explode(state): + if state != 'before get_docs': + return + raise l2db.errors.U1DBError("fail") + self.set_trace_hook(before_get_docs_explode) + # suppress traceback printing in the wsgiref server + # self.patch(simple_server.ServerHandler, + # 'log_exception', lambda h, exc_info: None) + doc = self.db.create_doc_from_json(tests.simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertRaises( + (l2db.errors.U1DBError, l2db.errors.BrokenSyncStream), + self.st.sync_exchange, [], 'other-replica', + last_known_generation=0, last_known_trans_id=None, + insert_doc_cb=self.receive_doc) + + @defer.inlineCallbacks + def test_sync_exchange_doc_ids(self): + sync_exchange_doc_ids = getattr(self.st, 'sync_exchange_doc_ids', None) + if sync_exchange_doc_ids is None: + self.skipTest("sync_exchange_doc_ids not implemented") + db2 = self.create_database('test2') + doc = db2.create_doc_from_json(tests.simple_doc) + new_gen, trans_id = yield sync_exchange_doc_ids( + db2, [(doc.doc_id, 10, 'T-sid')], 0, None, + insert_doc_cb=self.receive_doc) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, + tests.simple_doc, False) + self.assertTransactionLog([doc.doc_id], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 1, last_trans_id), + (self.other_changes, new_gen, trans_id)) + self.assertEqual(10, self.st.get_sync_info(db2._replica_uid)[3]) + + @defer.inlineCallbacks + def test__set_trace_hook(self): + called = [] + + def cb(state): + called.append(state) + + self.set_trace_hook(cb) + yield self.st.sync_exchange([], 'replica', 0, None, self.receive_doc) + yield self.st.record_sync_info('replica', 0, 'T-sid') + self.assertEqual(['before whats_changed', + 'after whats_changed', + 'before get_docs', + 'record_sync_info', + ], + called) + + @defer.inlineCallbacks + def test__set_trace_hook_shallow(self): + if (self.st._set_trace_hook_shallow == self.st._set_trace_hook or + self.st._set_trace_hook_shallow.im_func == + target.SoledadHTTPSyncTarget._set_trace_hook_shallow.im_func): + # shallow same as full + expected = ['before whats_changed', + 'after whats_changed', + 'before get_docs', + 'record_sync_info', + ] + else: + expected = ['sync_exchange', 'record_sync_info'] + + called = [] + + def cb(state): + called.append(state) + + self.set_trace_hook(cb, shallow=True) + yield self.st.sync_exchange([], 'replica', 0, None, self.receive_doc) + yield self.st.record_sync_info('replica', 0, 'T-sid') + self.assertEqual(expected, called) + + +# Just to make clear how this test is different... :) +DEFER_DECRYPTION = False + +WAIT_STEP = 1 +MAX_WAIT = 10 +DBPASS = "pass" + + +class SyncTimeoutError(Exception): + + """ + Dummy exception to notify timeout during sync. + """ + pass + + +class TestSoledadDbSync( + TestWithScenarios, + SoledadWithCouchServerMixin, + tests.TestCaseWithServer): + + """Test db.sync remote sync shortcut""" + + scenarios = [ + ('py-token-http', { + 'create_db_and_target': make_local_db_and_token_soledad_target, + 'make_app_with_state': make_token_soledad_app, + 'make_database_for_test': make_sqlcipher_database_for_test, + 'token': True + }), + ] + + oauth = False + token = False + + def setUp(self): + """ + Need to explicitely invoke inicialization on all bases. + """ + SoledadWithCouchServerMixin.setUp(self) + self.server = self.server_thread = None + self.startTwistedServer() + self.syncer = None + + # config info + self.db1_file = os.path.join(self.tempdir, "db1.u1db") + os.unlink(self.db1_file) + self.db_pass = DBPASS + self.email = ADDRESS + + # get a random prefix for each test, so we do not mess with + # concurrency during initialization and shutting down of + # each local db. + self.rand_prefix = ''.join( + map(lambda x: random.choice(string.ascii_letters), range(6))) + + # open test dbs: db1 will be the local sqlcipher db (which + # instantiates a syncdb). We use the self._soledad instance that was + # already created on some setUp method. + import binascii + tohex = binascii.b2a_hex + key = tohex(self._soledad.secrets.get_local_storage_key()) + sync_db_key = tohex(self._soledad.secrets.get_sync_db_key()) + dbpath = self._soledad._local_db_path + + self.opts = SQLCipherOptions( + dbpath, key, is_raw_key=True, create=False, + defer_encryption=True, sync_db_key=sync_db_key) + self.db1 = SQLCipherDatabase(self.opts) + + self.db2 = self.request_state._create_database(replica_uid='test') + + def tearDown(self): + """ + Need to explicitely invoke destruction on all bases. + """ + dbsyncer = getattr(self, 'dbsyncer', None) + if dbsyncer: + dbsyncer.close() + self.db1.close() + self.db2.close() + self._soledad.close() + + # XXX should not access "private" attrs + shutil.rmtree(os.path.dirname(self._soledad._local_db_path)) + SoledadWithCouchServerMixin.tearDown(self) + + def do_sync(self, target_name): + """ + Perform sync using SoledadSynchronizer, SoledadSyncTarget + and Token auth. + """ + if self.token: + creds = {'token': { + 'uuid': 'user-uuid', + 'token': 'auth-token', + }} + target_url = self.getURL(self.db2._dbname) + + # get a u1db syncer + crypto = self._soledad._crypto + replica_uid = self.db1._replica_uid + dbsyncer = SQLCipherU1DBSync( + self.opts, + crypto, + replica_uid, + None, + defer_encryption=True) + self.dbsyncer = dbsyncer + return dbsyncer.sync(target_url, + creds=creds, + defer_decryption=DEFER_DECRYPTION) + else: + return self._do_sync(self, target_name) + + def _do_sync(self, target_name): + if self.oauth: + path = '~/' + target_name + extra = dict(creds={'oauth': { + 'consumer_key': tests.consumer1.key, + 'consumer_secret': tests.consumer1.secret, + 'token_key': tests.token1.key, + 'token_secret': tests.token1.secret, + }}) + else: + path = target_name + extra = {} + target_url = self.getURL(path) + return self.db.sync(target_url, **extra) + + def wait_for_sync(self): + """ + Wait for sync to finish. + """ + wait = 0 + syncer = self.syncer + if syncer is not None: + while syncer.syncing: + time.sleep(WAIT_STEP) + wait += WAIT_STEP + if wait >= MAX_WAIT: + raise SyncTimeoutError + + def test_db_sync(self): + """ + Test sync. + + Adapted to check for encrypted content. + """ + doc1 = self.db1.create_doc_from_json(tests.simple_doc) + doc2 = self.db2.create_doc_from_json(tests.nested_doc) + d = self.do_sync('test') + + def _assert_successful_sync(results): + import time + # need to give time to the encryption to proceed + # TODO should implement a defer list to subscribe to the + # all-decrypted event + time.sleep(2) + local_gen_before_sync = results + self.wait_for_sync() + + gen, _, changes = self.db1.whats_changed(local_gen_before_sync) + self.assertEqual(1, len(changes)) + + self.assertEqual(doc2.doc_id, changes[0][0]) + self.assertEqual(1, gen - local_gen_before_sync) + + self.assertGetEncryptedDoc( + self.db2, doc1.doc_id, doc1.rev, tests.simple_doc, False) + self.assertGetEncryptedDoc( + self.db1, doc2.doc_id, doc2.rev, tests.nested_doc, False) + + d.addCallback(_assert_successful_sync) + return d + + +class SQLCipherSyncTargetTests(SoledadDatabaseSyncTargetTests): + + # TODO: implement _set_trace_hook(_shallow) in SoledadHTTPSyncTarget so + # skipped tests can be succesfully executed. + + scenarios = (tests.multiply_scenarios(SQLCIPHER_SCENARIOS, + target_scenarios)) + + whitebox = False diff --git a/testing/tox.ini b/testing/tox.ini new file mode 100644 index 00000000..0a8dda9d --- /dev/null +++ b/testing/tox.ini @@ -0,0 +1,21 @@ +[tox] +envlist = py27 + +[testenv] +commands = py.test {posargs} +changedir = tests +deps = + pytest + mock + testscenarios + setuptools-trial + pep8 + pdbpp + couchdb +# install soledad local packages + -e../common + -e../client + -e../server +setenv = + HOME=/tmp +install_command = pip install {opts} {packages} -- cgit v1.2.3 From c169046ad5e7888211a5359a0d8289efd91c9f9c Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 7 Jul 2016 11:51:14 +0200 Subject: [pkg] add u1db dependencies directly in leap.soledad.common --- common/pkg/requirements.pip | 2 ++ 1 file changed, 2 insertions(+) diff --git a/common/pkg/requirements.pip b/common/pkg/requirements.pip index e69de29b..d3ed2b50 100644 --- a/common/pkg/requirements.pip +++ b/common/pkg/requirements.pip @@ -0,0 +1,2 @@ +paste +routes -- cgit v1.2.3 From 59afc1eaf4136849f70ebadd27f01e67f4b894d3 Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 7 Jul 2016 11:52:57 +0200 Subject: [pkg] remove unneeded oauth code --- .../leap/soledad/common/l2db/remote/http_client.py | 43 +++------------------- 1 file changed, 5 insertions(+), 38 deletions(-) diff --git a/common/src/leap/soledad/common/l2db/remote/http_client.py b/common/src/leap/soledad/common/l2db/remote/http_client.py index eea42888..a65264b6 100644 --- a/common/src/leap/soledad/common/l2db/remote/http_client.py +++ b/common/src/leap/soledad/common/l2db/remote/http_client.py @@ -1,5 +1,4 @@ # Copyright 2011-2012 Canonical Ltd. -# Copyright 2016 LEAP Encryption Access Project # # This file is part of u1db. # @@ -18,7 +17,6 @@ """Base class to make requests to a remote HTTP server.""" import httplib -from oauth import oauth try: import simplejson as json except ImportError: @@ -33,9 +31,7 @@ from time import sleep from leap.soledad.common.l2db import errors from leap.soledad.common.l2db.remote import http_errors -from leap.soledad.common.l2db.remote.ssl_match_hostname import ( # noqa - CertificateError, - match_hostname) +from leap.soledad.common.l2db.remote.ssl_match_hostname import match_hostname # Ubuntu/debian # XXX other... @@ -68,7 +64,7 @@ class _VerifiedHTTPSConnection(httplib.HTTPSConnection): cert_opts = { 'cert_reqs': ssl.CERT_REQUIRED, 'ca_certs': CA_CERTS - } + } else: # XXX no cert verification implemented elsewhere for now cert_opts = {} @@ -83,13 +79,6 @@ class _VerifiedHTTPSConnection(httplib.HTTPSConnection): class HTTPClientBase(object): """Base class to make requests to a remote HTTP server.""" - # by default use HMAC-SHA1 OAuth signature method to not disclose - # tokens - # NB: given that the content bodies are not covered by the - # signatures though, to achieve security (against man-in-the-middle - # attacks for example) one would need HTTPS - oauth_signature_method = oauth.OAuthSignatureMethod_HMAC_SHA1() - # Will use these delays to retry on 503 befor finally giving up. The final # 0 is there to not wait after the final try fails. _delays = (1, 1, 2, 4, 0) @@ -108,12 +97,6 @@ class HTTPClientBase(object): raise errors.UnknownAuthMethod(auth_meth) set_creds(**credentials) - def set_oauth_credentials(self, consumer_key, consumer_secret, - token_key, token_secret): - self._creds = {'oauth': ( - oauth.OAuthConsumer(consumer_key, consumer_secret), - oauth.OAuthToken(token_key, token_secret))} - def _ensure_connection(self): if self._conn is not None: return @@ -149,7 +132,6 @@ class HTTPClientBase(object): except ValueError: pass else: - print "ERROR--->", respdic self._error(respdic) # special case if resp.status == 503: @@ -157,25 +139,10 @@ class HTTPClientBase(object): raise errors.HTTPError(resp.status, body, headers) def _sign_request(self, method, url_query, params): - if 'oauth' in self._creds: - consumer, token = self._creds['oauth'] - full_url = "%s://%s%s" % (self._url.scheme, self._url.netloc, - url_query) - oauth_req = oauth.OAuthRequest.from_consumer_and_token( - consumer, token, - http_method=method, - parameters=params, - http_url=full_url - ) - oauth_req.sign_request( - self.oauth_signature_method, consumer, token) - # Authorization: OAuth ... - return oauth_req.to_header().items() - else: - return [] + raise NotImplementedError def _request(self, method, url_parts, params=None, body=None, - content_type=None): + content_type=None): self._ensure_connection() unquoted_url = url_query = self._url.path if url_parts: @@ -209,7 +176,7 @@ class HTTPClientBase(object): raise e def _request_json(self, method, url_parts, params=None, body=None, - content_type=None): + content_type=None): res, headers = self._request(method, url_parts, params, body, content_type) return json.loads(res), headers -- cgit v1.2.3 From 41f65e3357011dbb7f510a6e87c6693bcc1d4edf Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 7 Jul 2016 11:53:45 +0200 Subject: [pkg] remove testing couchdb dep from common --- common/setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/common/setup.py b/common/setup.py index c1f4d5ac..7191fa00 100644 --- a/common/setup.py +++ b/common/setup.py @@ -284,11 +284,9 @@ setup( namespace_packages=["leap", "leap.soledad"], packages=find_packages('src', exclude=['*.tests', '*.tests.*']), package_dir={'': 'src'}, + package_data={'': ["*.sql"]}, test_suite='leap.soledad.common.tests', install_requires=requirements, tests_require=utils.parse_requirements( reqfiles=['pkg/requirements-testing.pip']), - extras_require={ - 'couchdb': ['couchdb'], - }, ) -- cgit v1.2.3 From 297ecdb24b238eff7e7674c7ab2df1f116007d7e Mon Sep 17 00:00:00 2001 From: drebs Date: Thu, 7 Jul 2016 11:56:33 +0200 Subject: [pkg] remove unneeded dirspec exceptions --- client/pkg/generate_wheels.sh | 2 +- client/pkg/pip_install_requirements.sh | 2 +- client/pkg/requirements-latest.pip | 2 -- client/pkg/requirements.pip | 1 - common/pkg/generate_wheels.sh | 2 +- common/pkg/pip_install_requirements.sh | 2 +- common/pkg/requirements-latest.pip | 1 - server/pkg/generate_wheels.sh | 2 +- server/pkg/pip_install_requirements.sh | 2 +- server/pkg/requirements-latest.pip | 2 -- 10 files changed, 6 insertions(+), 12 deletions(-) diff --git a/client/pkg/generate_wheels.sh b/client/pkg/generate_wheels.sh index 496f8e01..a13e2c7a 100755 --- a/client/pkg/generate_wheels.sh +++ b/client/pkg/generate_wheels.sh @@ -7,7 +7,7 @@ if [ "$WHEELHOUSE" = "" ]; then fi pip wheel --wheel-dir $WHEELHOUSE pip -pip wheel --wheel-dir $WHEELHOUSE --allow-external dirspec --allow-unverified dirspec -r pkg/requirements.pip +pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements.pip if [ -f pkg/requirements-testing.pip ]; then pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements-testing.pip fi diff --git a/client/pkg/pip_install_requirements.sh b/client/pkg/pip_install_requirements.sh index b97c826f..f4b5f67a 100755 --- a/client/pkg/pip_install_requirements.sh +++ b/client/pkg/pip_install_requirements.sh @@ -4,7 +4,7 @@ # Use at your own risk. # See $usage for help -insecure_packages="dirspec" +insecure_packages="" leap_wheelhouse=https://lizard.leap.se/wheels show_help() { diff --git a/client/pkg/requirements-latest.pip b/client/pkg/requirements-latest.pip index fa483db7..46a7ccba 100644 --- a/client/pkg/requirements-latest.pip +++ b/client/pkg/requirements-latest.pip @@ -1,7 +1,5 @@ --index-url https://pypi.python.org/simple/ ---allow-external dirspec --allow-unverified dirspec - -e 'git+https://github.com/pixelated-project/leap_pycommon.git@develop#egg=leap.common' -e '../common' -e . diff --git a/client/pkg/requirements.pip b/client/pkg/requirements.pip index 9596470f..2ae844e1 100644 --- a/client/pkg/requirements.pip +++ b/client/pkg/requirements.pip @@ -2,4 +2,3 @@ pysqlcipher>2.6.3 scrypt zope.proxy twisted - diff --git a/common/pkg/generate_wheels.sh b/common/pkg/generate_wheels.sh index 496f8e01..a13e2c7a 100755 --- a/common/pkg/generate_wheels.sh +++ b/common/pkg/generate_wheels.sh @@ -7,7 +7,7 @@ if [ "$WHEELHOUSE" = "" ]; then fi pip wheel --wheel-dir $WHEELHOUSE pip -pip wheel --wheel-dir $WHEELHOUSE --allow-external dirspec --allow-unverified dirspec -r pkg/requirements.pip +pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements.pip if [ -f pkg/requirements-testing.pip ]; then pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements-testing.pip fi diff --git a/common/pkg/pip_install_requirements.sh b/common/pkg/pip_install_requirements.sh index b97c826f..f4b5f67a 100755 --- a/common/pkg/pip_install_requirements.sh +++ b/common/pkg/pip_install_requirements.sh @@ -4,7 +4,7 @@ # Use at your own risk. # See $usage for help -insecure_packages="dirspec" +insecure_packages="" leap_wheelhouse=https://lizard.leap.se/wheels show_help() { diff --git a/common/pkg/requirements-latest.pip b/common/pkg/requirements-latest.pip index 9b579503..396d77f1 100644 --- a/common/pkg/requirements-latest.pip +++ b/common/pkg/requirements-latest.pip @@ -1,5 +1,4 @@ --index-url https://pypi.python.org/simple/ ---allow-external dirspec --allow-unverified dirspec -e 'git+https://github.com/pixelated-project/leap_pycommon.git@develop#egg=leap.common' -e . diff --git a/server/pkg/generate_wheels.sh b/server/pkg/generate_wheels.sh index 496f8e01..a13e2c7a 100755 --- a/server/pkg/generate_wheels.sh +++ b/server/pkg/generate_wheels.sh @@ -7,7 +7,7 @@ if [ "$WHEELHOUSE" = "" ]; then fi pip wheel --wheel-dir $WHEELHOUSE pip -pip wheel --wheel-dir $WHEELHOUSE --allow-external dirspec --allow-unverified dirspec -r pkg/requirements.pip +pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements.pip if [ -f pkg/requirements-testing.pip ]; then pip wheel --wheel-dir $WHEELHOUSE -r pkg/requirements-testing.pip fi diff --git a/server/pkg/pip_install_requirements.sh b/server/pkg/pip_install_requirements.sh index b97c826f..f4b5f67a 100755 --- a/server/pkg/pip_install_requirements.sh +++ b/server/pkg/pip_install_requirements.sh @@ -4,7 +4,7 @@ # Use at your own risk. # See $usage for help -insecure_packages="dirspec" +insecure_packages="" leap_wheelhouse=https://lizard.leap.se/wheels show_help() { diff --git a/server/pkg/requirements-latest.pip b/server/pkg/requirements-latest.pip index fa483db7..46a7ccba 100644 --- a/server/pkg/requirements-latest.pip +++ b/server/pkg/requirements-latest.pip @@ -1,7 +1,5 @@ --index-url https://pypi.python.org/simple/ ---allow-external dirspec --allow-unverified dirspec - -e 'git+https://github.com/pixelated-project/leap_pycommon.git@develop#egg=leap.common' -e '../common' -e . -- cgit v1.2.3 From 42521e368734d358c3169495003fae32c28da2b1 Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 8 Jul 2016 13:02:10 +0200 Subject: [test] properly close dbpool on async test --- testing/tests/client/test_async.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/testing/tests/client/test_async.py b/testing/tests/client/test_async.py index 2ff70864..42c315fe 100644 --- a/testing/tests/client/test_async.py +++ b/testing/tests/client/test_async.py @@ -32,6 +32,14 @@ class ASyncSQLCipherRetryTestCase(BaseSoledadTest): NUM_DOCS = 5000 + def setUp(self): + BaseSoledadTest.setUp(self) + self._dbpool = self._get_dbpool() + + def tearDown(self): + self._dbpool.close() + BaseSoledadTest.tearDown(self) + def _get_dbpool(self): tmpdb = os.path.join(self.tempdir, "test.soledad") opts = SQLCipherOptions(tmpdb, "secret", create=True) @@ -72,10 +80,8 @@ class ASyncSQLCipherRetryTestCase(BaseSoledadTest): adbapi.SQLCIPHER_CONNECTION_TIMEOUT = 1 adbapi.SQLCIPHER_MAX_RETRIES = 1 - dbpool = self._get_dbpool() - def _create_doc(doc): - return dbpool.runU1DBQuery("create_doc", doc) + return self._dbpool.runU1DBQuery("create_doc", doc) def _insert_docs(): deferreds = [] @@ -95,7 +101,7 @@ class ASyncSQLCipherRetryTestCase(BaseSoledadTest): raise Exception d = _insert_docs() - d.addCallback(lambda _: dbpool.runU1DBQuery("get_all_docs")) + d.addCallback(lambda _: self._dbpool.runU1DBQuery("get_all_docs")) d.addErrback(_errback) return d @@ -115,10 +121,8 @@ class ASyncSQLCipherRetryTestCase(BaseSoledadTest): above will fail and we should remove this comment from here. """ - dbpool = self._get_dbpool() - def _create_doc(doc): - return dbpool.runU1DBQuery("create_doc", doc) + return self._dbpool.runU1DBQuery("create_doc", doc) def _insert_docs(): deferreds = [] @@ -137,6 +141,6 @@ class ASyncSQLCipherRetryTestCase(BaseSoledadTest): raise Exception d = _insert_docs() - d.addCallback(lambda _: dbpool.runU1DBQuery("get_all_docs")) + d.addCallback(lambda _: self._dbpool.runU1DBQuery("get_all_docs")) d.addCallback(_count_docs) return d -- cgit v1.2.3 From d99198046e07abb0d19fde1695d22267bc5e1433 Mon Sep 17 00:00:00 2001 From: drebs Date: Fri, 8 Jul 2016 13:09:26 +0200 Subject: [bug] properly trap db errors and close resources SQLCipher database access errors can raise Soledad exceptions. Database access and multithreading resources are allocated in different places, so we have to be careful to close all multithreading mechanismis in case of database access errors. If we don't, zombie threads may haunt the reactor. This commit adds SQLCipher exception trapping and Soledad exception raising for database access errors, while properly shutting down multithreading resources. --- client/src/leap/soledad/client/adbapi.py | 29 ++++++++++++++++++----------- client/src/leap/soledad/client/api.py | 19 ++++++++++++++++--- client/src/leap/soledad/client/sqlcipher.py | 27 ++++++++++++++------------- testing/test_soledad/util.py | 2 +- 4 files changed, 49 insertions(+), 28 deletions(-) diff --git a/client/src/leap/soledad/client/adbapi.py b/client/src/leap/soledad/client/adbapi.py index 328b4762..234be6b6 100644 --- a/client/src/leap/soledad/client/adbapi.py +++ b/client/src/leap/soledad/client/adbapi.py @@ -29,8 +29,7 @@ from twisted.enterprise import adbapi from twisted.internet.defer import DeferredSemaphore from twisted.python import log from zope.proxy import ProxyBase, setProxiedObject -from pysqlcipher.dbapi2 import OperationalError -from pysqlcipher.dbapi2 import DatabaseError +from pysqlcipher import dbapi2 from leap.soledad.common.errors import DatabaseAccessError @@ -105,8 +104,10 @@ class U1DBConnection(adbapi.Connection): self._sync_enc_pool = sync_enc_pool try: adbapi.Connection.__init__(self, pool) - except DatabaseError: - raise DatabaseAccessError('Could not open sqlcipher database') + except dbapi2.DatabaseError as e: + raise DatabaseAccessError( + 'Error initializing connection to sqlcipher database: %s' + % str(e)) def reconnect(self): """ @@ -174,8 +175,9 @@ class U1DBConnectionPool(adbapi.ConnectionPool): self._sync_enc_pool = kwargs.pop("sync_enc_pool") try: adbapi.ConnectionPool.__init__(self, *args, **kwargs) - except DatabaseError: - raise DatabaseAccessError('Could not open sqlcipher database') + except dbapi2.DatabaseError as e: + raise DatabaseAccessError( + 'Error initializing u1db connection pool: %s' % str(e)) # all u1db connections, hashed by thread-id self._u1dbconnections = {} @@ -183,10 +185,15 @@ class U1DBConnectionPool(adbapi.ConnectionPool): # The replica uid, primed by the connections on init. self.replica_uid = ProxyBase(None) - conn = self.connectionFactory( - self, self._sync_enc_pool, init_u1db=True) - replica_uid = conn._u1db._real_replica_uid - setProxiedObject(self.replica_uid, replica_uid) + try: + conn = self.connectionFactory( + self, self._sync_enc_pool, init_u1db=True) + replica_uid = conn._u1db._real_replica_uid + setProxiedObject(self.replica_uid, replica_uid) + except DatabaseAccessError as e: + self.threadpool.stop() + raise DatabaseAccessError( + "Error initializing connection factory: %s" % str(e)) def runU1DBQuery(self, meth, *args, **kw): """ @@ -211,7 +218,7 @@ class U1DBConnectionPool(adbapi.ConnectionPool): self._runU1DBQuery, meth, *args, **kw) def _errback(failure): - failure.trap(OperationalError) + failure.trap(dbapi2.OperationalError) if failure.getErrorMessage() == "database is locked": should_retry = semaphore.acquire() if should_retry: diff --git a/client/src/leap/soledad/client/api.py b/client/src/leap/soledad/client/api.py index 8c25243b..1bfbed8a 100644 --- a/client/src/leap/soledad/client/api.py +++ b/client/src/leap/soledad/client/api.py @@ -51,6 +51,7 @@ from leap.soledad.common import soledad_assert from leap.soledad.common import soledad_assert_type from leap.soledad.common.l2db.remote import http_client from leap.soledad.common.l2db.remote.ssl_match_hostname import match_hostname +from leap.soledad.common.errors import DatabaseAccessError from leap.soledad.client import adbapi from leap.soledad.client import events as soledad_events @@ -213,10 +214,22 @@ class Soledad(object): self._init_secrets() self._crypto = SoledadCrypto(self._secrets.remote_storage_secret) - self._init_u1db_sqlcipher_backend() - if syncable: - self._init_u1db_syncer() + try: + # initialize database access, trap any problems so we can shutdown + # smoothly. + self._init_u1db_sqlcipher_backend() + if syncable: + self._init_u1db_syncer() + except DatabaseAccessError: + # oops! something went wrong with backend initialization. We + # have to close any thread-related stuff we have already opened + # here, otherwise there might be zombie threads that may clog the + # reactor. + self._sync_db.close() + if hasattr(self, '_dbpool'): + self._dbpool.close() + raise # # initialization/destruction methods diff --git a/client/src/leap/soledad/client/sqlcipher.py b/client/src/leap/soledad/client/sqlcipher.py index f36c0b6a..166c0783 100644 --- a/client/src/leap/soledad/client/sqlcipher.py +++ b/client/src/leap/soledad/client/sqlcipher.py @@ -58,6 +58,7 @@ from leap.soledad.common.document import SoledadDocument from leap.soledad.common import l2db from leap.soledad.common.l2db import errors as u1db_errors from leap.soledad.common.l2db.backends import sqlite_backend +from leap.soledad.common.errors import DatabaseAccessError from leap.soledad.client.http_target import SoledadHTTPSyncTarget from leap.soledad.client.sync import SoledadSynchronizer @@ -442,22 +443,19 @@ class SQLCipherU1DBSync(SQLCipherDatabase): # format is the following: # # self._syncers = {'': ('', syncer), ...} - self._syncers = {} - - # Storage for the documents received during a sync + # storage for the documents received during a sync self.received_docs = [] self.running = False + self.shutdownID = None + self._db_handle = None + # initialize the main db before scheduling a start + self._initialize_main_db() self._reactor = reactor self._reactor.callWhenRunning(self._start) - self._db_handle = None - self._initialize_main_db() - - self.shutdownID = None - if DO_STATS: self.sync_phase = None @@ -472,11 +470,14 @@ class SQLCipherU1DBSync(SQLCipherDatabase): self.running = True def _initialize_main_db(self): - self._db_handle = initialize_sqlcipher_db( - self._opts, check_same_thread=False) - self._real_replica_uid = None - self._ensure_schema() - self.set_document_factory(soledad_doc_factory) + try: + self._db_handle = initialize_sqlcipher_db( + self._opts, check_same_thread=False) + self._real_replica_uid = None + self._ensure_schema() + self.set_document_factory(soledad_doc_factory) + except sqlcipher_dbapi2.DatabaseError as e: + raise DatabaseAccessError(str(e)) @defer.inlineCallbacks def sync(self, url, creds=None, defer_decryption=True): diff --git a/testing/test_soledad/util.py b/testing/test_soledad/util.py index f81001b9..033a55df 100644 --- a/testing/test_soledad/util.py +++ b/testing/test_soledad/util.py @@ -313,7 +313,7 @@ class BaseSoledadTest(BaseLeapTest, MockedSharedDBTest): self.tempdir, prefix, secrets_path), local_db_path=os.path.join( self.tempdir, prefix, local_db_path), - server_url=server_url, # Soledad will fail if not given an url. + server_url=server_url, # Soledad will fail if not given an url cert_file=cert_file, defer_encryption=self.defer_sync_encryption, shared_db=MockSharedDB(), -- cgit v1.2.3 From f406ccbaf2b79db1d65827463f830f4ffbe5856c Mon Sep 17 00:00:00 2001 From: drebs Date: Sun, 10 Jul 2016 10:38:04 +0200 Subject: [refactor] make u1db connection pool args explicit --- client/src/leap/soledad/client/adbapi.py | 16 +- testing/tests/client/test_async.py | 146 ------ testing/tests/client/test_sqlcipher.py | 705 --------------------------- testing/tests/client/test_sqlcipher_sync.py | 730 ---------------------------- testing/tests/sqlcipher/test_async.py | 146 ++++++ testing/tests/sqlcipher/test_sqlcipher.py | 705 +++++++++++++++++++++++++++ testing/tests/sync/test_sqlcipher_sync.py | 730 ++++++++++++++++++++++++++++ 7 files changed, 1590 insertions(+), 1588 deletions(-) delete mode 100644 testing/tests/client/test_async.py delete mode 100644 testing/tests/client/test_sqlcipher.py delete mode 100644 testing/tests/client/test_sqlcipher_sync.py create mode 100644 testing/tests/sqlcipher/test_async.py create mode 100644 testing/tests/sqlcipher/test_sqlcipher.py create mode 100644 testing/tests/sync/test_sqlcipher_sync.py diff --git a/client/src/leap/soledad/client/adbapi.py b/client/src/leap/soledad/client/adbapi.py index 234be6b6..ef0f9066 100644 --- a/client/src/leap/soledad/client/adbapi.py +++ b/client/src/leap/soledad/client/adbapi.py @@ -78,9 +78,11 @@ def getConnectionPool(opts, openfun=None, driver="pysqlcipher", if openfun is None and driver == "pysqlcipher": openfun = partial(set_init_pragmas, opts=opts) return U1DBConnectionPool( - "%s.dbapi2" % driver, opts=opts, sync_enc_pool=sync_enc_pool, - database=opts.path, check_same_thread=False, cp_openfun=openfun, - timeout=SQLCIPHER_CONNECTION_TIMEOUT) + opts, sync_enc_pool, + # the following params are relayed "as is" to twisted's + # ConnectionPool. + "%s.dbapi2" % driver, opts.path, timeout=SQLCIPHER_CONNECTION_TIMEOUT, + check_same_thread=False, cp_openfun=openfun) class U1DBConnection(adbapi.Connection): @@ -166,13 +168,12 @@ class U1DBConnectionPool(adbapi.ConnectionPool): connectionFactory = U1DBConnection transactionFactory = U1DBTransaction - def __init__(self, *args, **kwargs): + def __init__(self, opts, sync_enc_pool, *args, **kwargs): """ Initialize the connection pool. """ - # extract soledad-specific objects from keyword arguments - self.opts = kwargs.pop("opts") - self._sync_enc_pool = kwargs.pop("sync_enc_pool") + self.opts = opts + self._sync_enc_pool = sync_enc_pool try: adbapi.ConnectionPool.__init__(self, *args, **kwargs) except dbapi2.DatabaseError as e: @@ -220,6 +221,7 @@ class U1DBConnectionPool(adbapi.ConnectionPool): def _errback(failure): failure.trap(dbapi2.OperationalError) if failure.getErrorMessage() == "database is locked": + logger.warning("Database operation timed out.") should_retry = semaphore.acquire() if should_retry: logger.warning( diff --git a/testing/tests/client/test_async.py b/testing/tests/client/test_async.py deleted file mode 100644 index 42c315fe..00000000 --- a/testing/tests/client/test_async.py +++ /dev/null @@ -1,146 +0,0 @@ -# -*- coding: utf-8 -*- -# test_async.py -# Copyright (C) 2013, 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -import os -import hashlib - -from twisted.internet import defer - -from test_soledad.util import BaseSoledadTest -from leap.soledad.client import adbapi -from leap.soledad.client.sqlcipher import SQLCipherOptions - - -class ASyncSQLCipherRetryTestCase(BaseSoledadTest): - - """ - Test asynchronous SQLCipher operation. - """ - - NUM_DOCS = 5000 - - def setUp(self): - BaseSoledadTest.setUp(self) - self._dbpool = self._get_dbpool() - - def tearDown(self): - self._dbpool.close() - BaseSoledadTest.tearDown(self) - - def _get_dbpool(self): - tmpdb = os.path.join(self.tempdir, "test.soledad") - opts = SQLCipherOptions(tmpdb, "secret", create=True) - return adbapi.getConnectionPool(opts) - - def _get_sample(self): - if not getattr(self, "_sample", None): - dirname = os.path.dirname(os.path.realpath(__file__)) - sample_file = os.path.join(dirname, "hacker_crackdown.txt") - with open(sample_file) as f: - self._sample = f.readlines() - return self._sample - - def test_concurrent_puts_fail_with_few_retries_and_small_timeout(self): - """ - Test if concurrent updates to the database with small timeout and - small number of retries fail with "database is locked" error. - - Many concurrent write attempts to the same sqlcipher database may fail - when the timeout is small and there are no retries. This test will - pass if any of the attempts to write the database fail. - - This test is much dependent on the environment and its result intends - to contrast with the test for the workaround for the "database is - locked" problem, which is addressed by the "test_concurrent_puts" test - below. - - If this test ever fails, it means that either (1) the platform where - you are running is it very powerful and you should try with an even - lower timeout value, or (2) the bug has been solved by a better - implementation of the underlying database pool, and thus this test - should be removed from the test suite. - """ - - old_timeout = adbapi.SQLCIPHER_CONNECTION_TIMEOUT - old_max_retries = adbapi.SQLCIPHER_MAX_RETRIES - - adbapi.SQLCIPHER_CONNECTION_TIMEOUT = 1 - adbapi.SQLCIPHER_MAX_RETRIES = 1 - - def _create_doc(doc): - return self._dbpool.runU1DBQuery("create_doc", doc) - - def _insert_docs(): - deferreds = [] - for i in range(self.NUM_DOCS): - payload = self._get_sample()[i] - chash = hashlib.sha256(payload).hexdigest() - doc = {"number": i, "payload": payload, 'chash': chash} - d = _create_doc(doc) - deferreds.append(d) - return defer.gatherResults(deferreds, consumeErrors=True) - - def _errback(e): - if e.value[0].getErrorMessage() == "database is locked": - adbapi.SQLCIPHER_CONNECTION_TIMEOUT = old_timeout - adbapi.SQLCIPHER_MAX_RETRIES = old_max_retries - return defer.succeed("") - raise Exception - - d = _insert_docs() - d.addCallback(lambda _: self._dbpool.runU1DBQuery("get_all_docs")) - d.addErrback(_errback) - return d - - def test_concurrent_puts(self): - """ - Test that many concurrent puts succeed. - - Currently, there's a known problem with the concurrent database pool - which is that many concurrent attempts to write to the database may - fail when the lock timeout is small and when there are no (or few) - retries. We currently workaround this problem by increasing the - timeout and the number of retries. - - Should this test ever fail, it probably means that the timeout and/or - number of retries should be increased for the platform you're running - the test. If the underlying database pool is ever fixed, then the test - above will fail and we should remove this comment from here. - """ - - def _create_doc(doc): - return self._dbpool.runU1DBQuery("create_doc", doc) - - def _insert_docs(): - deferreds = [] - for i in range(self.NUM_DOCS): - payload = self._get_sample()[i] - chash = hashlib.sha256(payload).hexdigest() - doc = {"number": i, "payload": payload, 'chash': chash} - d = _create_doc(doc) - deferreds.append(d) - return defer.gatherResults(deferreds, consumeErrors=True) - - def _count_docs(results): - _, docs = results - if self.NUM_DOCS == len(docs): - return defer.succeed("") - raise Exception - - d = _insert_docs() - d.addCallback(lambda _: self._dbpool.runU1DBQuery("get_all_docs")) - d.addCallback(_count_docs) - return d diff --git a/testing/tests/client/test_sqlcipher.py b/testing/tests/client/test_sqlcipher.py deleted file mode 100644 index 11472d46..00000000 --- a/testing/tests/client/test_sqlcipher.py +++ /dev/null @@ -1,705 +0,0 @@ -# -*- coding: utf-8 -*- -# test_sqlcipher.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Test sqlcipher backend internals. -""" -import os -import time -import threading -import tempfile -import shutil - -from pysqlcipher import dbapi2 -from testscenarios import TestWithScenarios - -# l2db stuff. -from leap.soledad.common.l2db import errors -from leap.soledad.common.l2db import query_parser -from leap.soledad.common.l2db.backends.sqlite_backend \ - import SQLitePartialExpandDatabase - -# soledad stuff. -from leap.soledad.common import soledad_assert -from leap.soledad.common.document import SoledadDocument -from leap.soledad.client.sqlcipher import SQLCipherDatabase -from leap.soledad.client.sqlcipher import SQLCipherOptions -from leap.soledad.client.sqlcipher import DatabaseIsNotEncrypted - -# u1db tests stuff. -from test_soledad import u1db_tests as tests -from test_soledad.u1db_tests import test_backends -from test_soledad.u1db_tests import test_open -from test_soledad.util import SQLCIPHER_SCENARIOS -from test_soledad.util import PASSWORD -from test_soledad.util import BaseSoledadTest - - -def sqlcipher_open(path, passphrase, create=True, document_factory=None): - return SQLCipherDatabase( - SQLCipherOptions(path, passphrase, create=create)) - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_common_backend`. -# ----------------------------------------------------------------------------- - -class TestSQLCipherBackendImpl(tests.TestCase): - - def test__allocate_doc_id(self): - db = sqlcipher_open(':memory:', PASSWORD) - doc_id1 = db._allocate_doc_id() - self.assertTrue(doc_id1.startswith('D-')) - self.assertEqual(34, len(doc_id1)) - int(doc_id1[len('D-'):], 16) - self.assertNotEqual(doc_id1, db._allocate_doc_id()) - db.close() - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_backends`. -# ----------------------------------------------------------------------------- - -class SQLCipherTests(TestWithScenarios, test_backends.AllDatabaseTests): - scenarios = SQLCIPHER_SCENARIOS - - -class SQLCipherDatabaseTests(TestWithScenarios, - test_backends.LocalDatabaseTests): - scenarios = SQLCIPHER_SCENARIOS - - -class SQLCipherValidateGenNTransIdTests( - TestWithScenarios, - test_backends.LocalDatabaseValidateGenNTransIdTests): - scenarios = SQLCIPHER_SCENARIOS - - -class SQLCipherValidateSourceGenTests( - TestWithScenarios, - test_backends.LocalDatabaseValidateSourceGenTests): - scenarios = SQLCIPHER_SCENARIOS - - -class SQLCipherWithConflictsTests( - TestWithScenarios, - test_backends.LocalDatabaseWithConflictsTests): - scenarios = SQLCIPHER_SCENARIOS - - -class SQLCipherIndexTests( - TestWithScenarios, test_backends.DatabaseIndexTests): - scenarios = SQLCIPHER_SCENARIOS - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_sqlite_backend`. -# ----------------------------------------------------------------------------- - -class TestSQLCipherDatabase(tests.TestCase): - """ - Tests from u1db.tests.test_sqlite_backend.TestSQLiteDatabase. - """ - - def test_atomic_initialize(self): - # This test was modified to ensure that db2.close() is called within - # the thread that created the database. - tmpdir = self.createTempDir() - dbname = os.path.join(tmpdir, 'atomic.db') - - t2 = None # will be a thread - - class SQLCipherDatabaseTesting(SQLCipherDatabase): - _index_storage_value = "testing" - - def __init__(self, dbname, ntry): - self._try = ntry - self._is_initialized_invocations = 0 - SQLCipherDatabase.__init__( - self, - SQLCipherOptions(dbname, PASSWORD)) - - def _is_initialized(self, c): - res = \ - SQLCipherDatabase._is_initialized(self, c) - if self._try == 1: - self._is_initialized_invocations += 1 - if self._is_initialized_invocations == 2: - t2.start() - # hard to do better and have a generic test - time.sleep(0.05) - return res - - class SecondTry(threading.Thread): - - outcome2 = [] - - def run(self): - try: - db2 = SQLCipherDatabaseTesting(dbname, 2) - except Exception, e: - SecondTry.outcome2.append(e) - else: - SecondTry.outcome2.append(db2) - - t2 = SecondTry() - db1 = SQLCipherDatabaseTesting(dbname, 1) - t2.join() - - self.assertIsInstance(SecondTry.outcome2[0], SQLCipherDatabaseTesting) - self.assertTrue(db1._is_initialized(db1._get_sqlite_handle().cursor())) - db1.close() - - -class TestSQLCipherPartialExpandDatabase(tests.TestCase): - """ - Tests from u1db.tests.test_sqlite_backend.TestSQLitePartialExpandDatabase. - """ - - # The following tests had to be cloned from u1db because they all - # instantiate the backend directly, so we need to change that in order to - # our backend be instantiated in place. - - def setUp(self): - self.db = sqlcipher_open(':memory:', PASSWORD) - - def tearDown(self): - self.db.close() - - def test_default_replica_uid(self): - self.assertIsNot(None, self.db._replica_uid) - self.assertEqual(32, len(self.db._replica_uid)) - int(self.db._replica_uid, 16) - - def test__parse_index(self): - g = self.db._parse_index_definition('fieldname') - self.assertIsInstance(g, query_parser.ExtractField) - self.assertEqual(['fieldname'], g.field) - - def test__update_indexes(self): - g = self.db._parse_index_definition('fieldname') - c = self.db._get_sqlite_handle().cursor() - self.db._update_indexes('doc-id', {'fieldname': 'val'}, - [('fieldname', g)], c) - c.execute('SELECT doc_id, field_name, value FROM document_fields') - self.assertEqual([('doc-id', 'fieldname', 'val')], - c.fetchall()) - - def test_create_database(self): - raw_db = self.db._get_sqlite_handle() - self.assertNotEqual(None, raw_db) - - def test__set_replica_uid(self): - # Start from scratch, so that replica_uid isn't set. - self.assertIsNot(None, self.db._real_replica_uid) - self.assertIsNot(None, self.db._replica_uid) - self.db._set_replica_uid('foo') - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT value FROM u1db_config WHERE name='replica_uid'") - self.assertEqual(('foo',), c.fetchone()) - self.assertEqual('foo', self.db._real_replica_uid) - self.assertEqual('foo', self.db._replica_uid) - self.db._close_sqlite_handle() - self.assertEqual('foo', self.db._replica_uid) - - def test__open_database(self): - # SQLCipherDatabase has no _open_database() method, so we just pass - # (and test for the same funcionality on test_open_database_existing() - # below). - pass - - def test__open_database_with_factory(self): - # SQLCipherDatabase has no _open_database() method. - pass - - def test__open_database_non_existent(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/non-existent.sqlite' - self.assertRaises(errors.DatabaseDoesNotExist, - sqlcipher_open, - path, PASSWORD, create=False) - - def test__open_database_during_init(self): - # The purpose of this test is to ensure that _open_database() parallel - # db initialization behaviour is correct. As SQLCipherDatabase does - # not have an _open_database() method, we just do not implement this - # test. - pass - - def test__open_database_invalid(self): - # This test was modified to ensure that an empty database file will - # raise a DatabaseIsNotEncrypted exception instead of a - # dbapi2.OperationalError exception. - temp_dir = self.createTempDir(prefix='u1db-test-') - path1 = temp_dir + '/invalid1.db' - with open(path1, 'wb') as f: - f.write("") - self.assertRaises(DatabaseIsNotEncrypted, - sqlcipher_open, path1, - PASSWORD) - with open(path1, 'wb') as f: - f.write("invalid") - self.assertRaises(dbapi2.DatabaseError, - sqlcipher_open, path1, - PASSWORD) - - def test_open_database_existing(self): - # In the context of SQLCipherDatabase, where no _open_database() - # method exists and thus there's no call to _which_index_storage(), - # this test tests for the same functionality as - # test_open_database_create() below. So, we just pass. - pass - - def test_open_database_with_factory(self): - # SQLCipherDatabase's constructor has no factory parameter. - pass - - def test_open_database_create(self): - # SQLCipherDatabas has no open_database() method, so we just test for - # the actual database constructor effects. - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/new.sqlite' - db1 = sqlcipher_open(path, PASSWORD, create=True) - db2 = sqlcipher_open(path, PASSWORD, create=False) - self.assertIsInstance(db2, SQLCipherDatabase) - db1.close() - db2.close() - - def test_create_database_initializes_schema(self): - # This test had to be cloned because our implementation of SQLCipher - # backend is referenced with an index_storage_value that includes the - # word "encrypted". See u1db's sqlite_backend and our - # sqlcipher_backend for reference. - raw_db = self.db._get_sqlite_handle() - c = raw_db.cursor() - c.execute("SELECT * FROM u1db_config") - config = dict([(r[0], r[1]) for r in c.fetchall()]) - replica_uid = self.db._replica_uid - self.assertEqual({'sql_schema': '0', 'replica_uid': replica_uid, - 'index_storage': 'expand referenced encrypted'}, - config) - - def test_store_syncable(self): - doc = self.db.create_doc_from_json(tests.simple_doc) - # assert that docs are syncable by default - self.assertEqual(True, doc.syncable) - # assert that we can store syncable = False - doc.syncable = False - self.db.put_doc(doc) - self.assertEqual(False, self.db.get_doc(doc.doc_id).syncable) - # assert that we can store syncable = True - doc.syncable = True - self.db.put_doc(doc) - self.assertEqual(True, self.db.get_doc(doc.doc_id).syncable) - - def test__close_sqlite_handle(self): - raw_db = self.db._get_sqlite_handle() - self.db._close_sqlite_handle() - self.assertRaises(dbapi2.ProgrammingError, - raw_db.cursor) - - def test__get_generation(self): - self.assertEqual(0, self.db._get_generation()) - - def test__get_generation_info(self): - self.assertEqual((0, ''), self.db._get_generation_info()) - - def test_create_index(self): - self.db.create_index('test-idx', "key") - self.assertEqual([('test-idx', ["key"])], self.db.list_indexes()) - - def test_create_index_multiple_fields(self): - self.db.create_index('test-idx', "key", "key2") - self.assertEqual([('test-idx', ["key", "key2"])], - self.db.list_indexes()) - - def test__get_index_definition(self): - self.db.create_index('test-idx', "key", "key2") - # TODO: How would you test that an index is getting used for an SQL - # request? - self.assertEqual(["key", "key2"], - self.db._get_index_definition('test-idx')) - - def test_list_index_mixed(self): - # Make sure that we properly order the output - c = self.db._get_sqlite_handle().cursor() - # We intentionally insert the data in weird ordering, to make sure the - # query still gets it back correctly. - c.executemany("INSERT INTO index_definitions VALUES (?, ?, ?)", - [('idx-1', 0, 'key10'), - ('idx-2', 2, 'key22'), - ('idx-1', 1, 'key11'), - ('idx-2', 0, 'key20'), - ('idx-2', 1, 'key21')]) - self.assertEqual([('idx-1', ['key10', 'key11']), - ('idx-2', ['key20', 'key21', 'key22'])], - self.db.list_indexes()) - - def test_no_indexes_no_document_fields(self): - self.db.create_doc_from_json( - '{"key1": "val1", "key2": "val2"}') - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([], c.fetchall()) - - def test_create_extracts_fields(self): - doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') - doc2 = self.db.create_doc_from_json('{"key1": "valx", "key2": "valy"}') - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([], c.fetchall()) - self.db.create_index('test', 'key1', 'key2') - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual(sorted( - [(doc1.doc_id, "key1", "val1"), - (doc1.doc_id, "key2", "val2"), - (doc2.doc_id, "key1", "valx"), - (doc2.doc_id, "key2", "valy"), ]), sorted(c.fetchall())) - - def test_put_updates_fields(self): - self.db.create_index('test', 'key1', 'key2') - doc1 = self.db.create_doc_from_json( - '{"key1": "val1", "key2": "val2"}') - doc1.content = {"key1": "val1", "key2": "valy"} - self.db.put_doc(doc1) - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([(doc1.doc_id, "key1", "val1"), - (doc1.doc_id, "key2", "valy"), ], c.fetchall()) - - def test_put_updates_nested_fields(self): - self.db.create_index('test', 'key', 'sub.doc') - doc1 = self.db.create_doc_from_json(tests.nested_doc) - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([(doc1.doc_id, "key", "value"), - (doc1.doc_id, "sub.doc", "underneath"), ], - c.fetchall()) - - def test__ensure_schema_rollback(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/rollback.db' - - class SQLitePartialExpandDbTesting(SQLCipherDatabase): - - def _set_replica_uid_in_transaction(self, uid): - super(SQLitePartialExpandDbTesting, - self)._set_replica_uid_in_transaction(uid) - if fail: - raise Exception() - - db = SQLitePartialExpandDbTesting.__new__(SQLitePartialExpandDbTesting) - db._db_handle = dbapi2.connect(path) # db is there but not yet init-ed - fail = True - self.assertRaises(Exception, db._ensure_schema) - fail = False - db._initialize(db._db_handle.cursor()) - - def test_open_database_non_existent(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/non-existent.sqlite' - self.assertRaises(errors.DatabaseDoesNotExist, - sqlcipher_open, path, "123", - create=False) - - def test_delete_database_existent(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/new.sqlite' - db = sqlcipher_open(path, "123", create=True) - db.close() - SQLCipherDatabase.delete_database(path) - self.assertRaises(errors.DatabaseDoesNotExist, - sqlcipher_open, path, "123", - create=False) - - def test_delete_database_nonexistent(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/non-existent.sqlite' - self.assertRaises(errors.DatabaseDoesNotExist, - SQLCipherDatabase.delete_database, path) - - def test__get_indexed_fields(self): - self.db.create_index('idx1', 'a', 'b') - self.assertEqual(set(['a', 'b']), self.db._get_indexed_fields()) - self.db.create_index('idx2', 'b', 'c') - self.assertEqual(set(['a', 'b', 'c']), self.db._get_indexed_fields()) - - def test_indexed_fields_expanded(self): - self.db.create_index('idx1', 'key1') - doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') - self.assertEqual(set(['key1']), self.db._get_indexed_fields()) - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([(doc1.doc_id, 'key1', 'val1')], c.fetchall()) - - def test_create_index_updates_fields(self): - doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') - self.db.create_index('idx1', 'key1') - self.assertEqual(set(['key1']), self.db._get_indexed_fields()) - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([(doc1.doc_id, 'key1', 'val1')], c.fetchall()) - - def assertFormatQueryEquals(self, exp_statement, exp_args, definition, - values): - statement, args = self.db._format_query(definition, values) - self.assertEqual(exp_statement, statement) - self.assertEqual(exp_args, args) - - def test__format_query(self): - self.assertFormatQueryEquals( - "SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM " - "document d, document_fields d0 LEFT OUTER JOIN conflicts c ON " - "c.doc_id = d.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name " - "= ? AND d0.value = ? GROUP BY d.doc_id, d.doc_rev, d.content " - "ORDER BY d0.value;", ["key1", "a"], - ["key1"], ["a"]) - - def test__format_query2(self): - self.assertFormatQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value = ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value = ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value = ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' - 'd0.value, d1.value, d2.value;', - ["key1", "a", "key2", "b", "key3", "c"], - ["key1", "key2", "key3"], ["a", "b", "c"]) - - def test__format_query_wildcard(self): - self.assertFormatQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value = ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value GLOB ? AND d.doc_id = d2.doc_id AND d2.field_name = ? ' - 'AND d2.value NOT NULL GROUP BY d.doc_id, d.doc_rev, d.content ' - 'ORDER BY d0.value, d1.value, d2.value;', - ["key1", "a", "key2", "b*", "key3"], ["key1", "key2", "key3"], - ["a", "b*", "*"]) - - def assertFormatRangeQueryEquals(self, exp_statement, exp_args, definition, - start_value, end_value): - statement, args = self.db._format_range_query( - definition, start_value, end_value) - self.assertEqual(exp_statement, statement) - self.assertEqual(exp_args, args) - - def test__format_range_query(self): - self.assertFormatRangeQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value >= ? AND d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value <= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value <= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' - 'd0.value, d1.value, d2.value;', - ['key1', 'a', 'key2', 'b', 'key3', 'c', 'key1', 'p', 'key2', 'q', - 'key3', 'r'], - ["key1", "key2", "key3"], ["a", "b", "c"], ["p", "q", "r"]) - - def test__format_range_query_no_start(self): - self.assertFormatRangeQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value <= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value <= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' - 'd0.value, d1.value, d2.value;', - ['key1', 'a', 'key2', 'b', 'key3', 'c'], - ["key1", "key2", "key3"], None, ["a", "b", "c"]) - - def test__format_range_query_no_end(self): - self.assertFormatRangeQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value >= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' - 'd0.value, d1.value, d2.value;', - ['key1', 'a', 'key2', 'b', 'key3', 'c'], - ["key1", "key2", "key3"], ["a", "b", "c"], None) - - def test__format_range_query_wildcard(self): - self.assertFormatRangeQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value NOT NULL AND d.doc_id = d0.doc_id AND d0.field_name = ? ' - 'AND d0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? ' - 'AND (d1.value < ? OR d1.value GLOB ?) AND d.doc_id = d2.doc_id ' - 'AND d2.field_name = ? AND d2.value NOT NULL GROUP BY d.doc_id, ' - 'd.doc_rev, d.content ORDER BY d0.value, d1.value, d2.value;', - ['key1', 'a', 'key2', 'b', 'key3', 'key1', 'p', 'key2', 'q', 'q*', - 'key3'], - ["key1", "key2", "key3"], ["a", "b*", "*"], ["p", "q*", "*"]) - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_open`. -# ----------------------------------------------------------------------------- - - -class SQLCipherOpen(test_open.TestU1DBOpen): - - def test_open_no_create(self): - self.assertRaises(errors.DatabaseDoesNotExist, - sqlcipher_open, self.db_path, - PASSWORD, - create=False) - self.assertFalse(os.path.exists(self.db_path)) - - def test_open_create(self): - db = sqlcipher_open(self.db_path, PASSWORD, create=True) - self.addCleanup(db.close) - self.assertTrue(os.path.exists(self.db_path)) - self.assertIsInstance(db, SQLCipherDatabase) - - def test_open_with_factory(self): - db = sqlcipher_open(self.db_path, PASSWORD, create=True, - document_factory=SoledadDocument) - self.addCleanup(db.close) - doc = db.create_doc({}) - self.assertTrue(isinstance(doc, SoledadDocument)) - - def test_open_existing(self): - db = sqlcipher_open(self.db_path, PASSWORD) - self.addCleanup(db.close) - doc = db.create_doc_from_json(tests.simple_doc) - # Even though create=True, we shouldn't wipe the db - db2 = sqlcipher_open(self.db_path, PASSWORD, create=True) - self.addCleanup(db2.close) - doc2 = db2.get_doc(doc.doc_id) - self.assertEqual(doc, doc2) - - def test_open_existing_no_create(self): - db = sqlcipher_open(self.db_path, PASSWORD) - self.addCleanup(db.close) - db2 = sqlcipher_open(self.db_path, PASSWORD, create=False) - self.addCleanup(db2.close) - self.assertIsInstance(db2, SQLCipherDatabase) - - -# ----------------------------------------------------------------------------- -# Tests for actual encryption of the database -# ----------------------------------------------------------------------------- - -class SQLCipherEncryptionTests(BaseSoledadTest): - - """ - Tests to guarantee SQLCipher is indeed encrypting data when storing. - """ - - def _delete_dbfiles(self): - for dbfile in [self.DB_FILE]: - if os.path.exists(dbfile): - os.unlink(dbfile) - - def setUp(self): - # the following come from BaseLeapTest.setUpClass, because - # twisted.trial doesn't support such class methods for setting up - # test classes. - self.old_path = os.environ['PATH'] - self.old_home = os.environ['HOME'] - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.home = self.tempdir - bin_tdir = os.path.join( - self.tempdir, - 'bin') - os.environ["PATH"] = bin_tdir - os.environ["HOME"] = self.tempdir - # this is our own stuff - self.DB_FILE = os.path.join(self.tempdir, 'test.db') - self._delete_dbfiles() - - def tearDown(self): - self._delete_dbfiles() - # the following come from BaseLeapTest.tearDownClass, because - # twisted.trial doesn't support such class methods for tearing down - # test classes. - os.environ["PATH"] = self.old_path - os.environ["HOME"] = self.old_home - # safety check! please do not wipe my home... - # XXX needs to adapt to non-linuces - soledad_assert( - self.tempdir.startswith('/tmp/leap_tests-') or - self.tempdir.startswith('/var/folder'), - "beware! tried to remove a dir which does not " - "live in temporal folder!") - shutil.rmtree(self.tempdir) - - def test_try_to_open_encrypted_db_with_sqlite_backend(self): - """ - SQLite backend should not succeed to open SQLCipher databases. - """ - db = sqlcipher_open(self.DB_FILE, PASSWORD) - doc = db.create_doc_from_json(tests.simple_doc) - db.close() - try: - # trying to open an encrypted database with the regular u1db - # backend should raise a DatabaseError exception. - SQLitePartialExpandDatabase(self.DB_FILE, - document_factory=SoledadDocument) - raise DatabaseIsNotEncrypted() - except dbapi2.DatabaseError: - # at this point we know that the regular U1DB sqlcipher backend - # did not succeed on opening the database, so it was indeed - # encrypted. - db = sqlcipher_open(self.DB_FILE, PASSWORD) - doc = db.get_doc(doc.doc_id) - self.assertEqual(tests.simple_doc, doc.get_json(), - 'decrypted content mismatch') - db.close() - - def test_try_to_open_raw_db_with_sqlcipher_backend(self): - """ - SQLCipher backend should not succeed to open unencrypted databases. - """ - db = SQLitePartialExpandDatabase(self.DB_FILE, - document_factory=SoledadDocument) - db.create_doc_from_json(tests.simple_doc) - db.close() - try: - # trying to open the a non-encrypted database with sqlcipher - # backend should raise a DatabaseIsNotEncrypted exception. - db = sqlcipher_open(self.DB_FILE, PASSWORD) - db.close() - raise dbapi2.DatabaseError( - "SQLCipher backend should not be able to open non-encrypted " - "dbs.") - except DatabaseIsNotEncrypted: - pass diff --git a/testing/tests/client/test_sqlcipher_sync.py b/testing/tests/client/test_sqlcipher_sync.py deleted file mode 100644 index 3cbefc8b..00000000 --- a/testing/tests/client/test_sqlcipher_sync.py +++ /dev/null @@ -1,730 +0,0 @@ -# -*- coding: utf-8 -*- -# test_sqlcipher.py -# Copyright (C) 2013-2016 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Test sqlcipher backend sync. -""" -import os - -from uuid import uuid4 - -from testscenarios import TestWithScenarios - -from leap.soledad.common.l2db import sync -from leap.soledad.common.l2db import vectorclock -from leap.soledad.common.l2db import errors - -from leap.soledad.common.crypto import ENC_SCHEME_KEY -from leap.soledad.client.crypto import decrypt_doc_dict -from leap.soledad.client.http_target import SoledadHTTPSyncTarget - -from test_soledad import u1db_tests as tests -from test_soledad.util import SQLCIPHER_SCENARIOS -from test_soledad.util import make_soledad_app -from test_soledad.util import soledad_sync_target -from test_soledad.util import BaseSoledadTest - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_sync`. -# ----------------------------------------------------------------------------- - -def sync_via_synchronizer_and_soledad(test, db_source, db_target, - trace_hook=None, - trace_hook_shallow=None): - if trace_hook: - test.skipTest("full trace hook unsupported over http") - path = test._http_at[db_target] - target = SoledadHTTPSyncTarget.connect( - test.getURL(path), test._soledad._crypto) - target.set_token_credentials('user-uuid', 'auth-token') - if trace_hook_shallow: - target._set_trace_hook_shallow(trace_hook_shallow) - return sync.Synchronizer(db_source, target).sync() - - -def sync_via_synchronizer(test, db_source, db_target, - trace_hook=None, - trace_hook_shallow=None): - target = db_target.get_sync_target() - trace_hook = trace_hook or trace_hook_shallow - if trace_hook: - target._set_trace_hook(trace_hook) - return sync.Synchronizer(db_source, target).sync() - - -sync_scenarios = [] -for name, scenario in SQLCIPHER_SCENARIOS: - scenario['do_sync'] = sync_via_synchronizer - sync_scenarios.append((name, scenario)) - - -class SQLCipherDatabaseSyncTests( - TestWithScenarios, - tests.DatabaseBaseTests, - BaseSoledadTest): - - """ - Test for succesfull sync between SQLCipher and LeapBackend. - - Some of the tests in this class had to be adapted because the remote - backend always receive encrypted content, and so it can not rely on - document's content comparison to try to autoresolve conflicts. - """ - - scenarios = sync_scenarios - - def setUp(self): - self._use_tracking = {} - super(tests.DatabaseBaseTests, self).setUp() - - def create_database(self, replica_uid, sync_role=None): - if replica_uid == 'test' and sync_role is None: - # created up the chain by base class but unused - return None - db = self.create_database_for_role(replica_uid, sync_role) - if sync_role: - self._use_tracking[db] = (replica_uid, sync_role) - self.addCleanup(db.close) - return db - - def create_database_for_role(self, replica_uid, sync_role): - # hook point for reuse - return tests.DatabaseBaseTests.create_database(self, replica_uid) - - def sync(self, db_from, db_to, trace_hook=None, - trace_hook_shallow=None): - from_name, from_sync_role = self._use_tracking[db_from] - to_name, to_sync_role = self._use_tracking[db_to] - if from_sync_role not in ('source', 'both'): - raise Exception("%s marked for %s use but used as source" % - (from_name, from_sync_role)) - if to_sync_role not in ('target', 'both'): - raise Exception("%s marked for %s use but used as target" % - (to_name, to_sync_role)) - return self.do_sync(self, db_from, db_to, trace_hook, - trace_hook_shallow) - - def assertLastExchangeLog(self, db, expected): - log = getattr(db, '_last_exchange_log', None) - if log is None: - return - self.assertEqual(expected, log) - - def copy_database(self, db, sync_role=None): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES - # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST - # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS - # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND - # NINJA TO YOUR HOUSE. - db_copy = tests.DatabaseBaseTests.copy_database(self, db) - name, orig_sync_role = self._use_tracking[db] - self._use_tracking[db_copy] = (name + '(copy)', sync_role or - orig_sync_role) - return db_copy - - def test_sync_tracks_db_generation_of_other(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.assertEqual(0, self.sync(self.db1, self.db2)) - self.assertEqual( - (0, ''), self.db1._get_replica_gen_and_trans_id('test2')) - self.assertEqual( - (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) - self.assertLastExchangeLog(self.db2, - {'receive': - {'docs': [], 'last_known_gen': 0}, - 'return': - {'docs': [], 'last_gen': 0}}) - - def test_sync_autoresolves(self): - """ - Test for sync autoresolve remote. - - This test was adapted because the remote database receives encrypted - content and so it can't compare documents contents to autoresolve. - """ - # The remote database can't autoresolve conflicts based on magic - # content convergence, so we modify this test to leave the possibility - # of the remode document ending up in conflicted state. - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc1 = self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc') - rev1 = doc1.rev - doc2 = self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc') - rev2 = doc2.rev - self.sync(self.db1, self.db2) - doc = self.db1.get_doc('doc') - self.assertFalse(doc.has_conflicts) - # if remote content is in conflicted state, then document revisions - # will be different. - # self.assertEqual(doc.rev, self.db2.get_doc('doc').rev) - v = vectorclock.VectorClockRev(doc.rev) - self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev1))) - self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev2))) - - def test_sync_autoresolves_moar(self): - """ - Test for sync autoresolve local. - - This test was adapted to decrypt remote content before assert. - """ - # here we test that when a database that has a conflicted document is - # the source of a sync, and the target database has a revision of the - # conflicted document that is newer than the source database's, and - # that target's database's document's content is the same as the - # source's document's conflict's, the source's document's conflict gets - # autoresolved, and the source's document's revision bumped. - # - # idea is as follows: - # A B - # a1 - - # `-------> - # a1 a1 - # v v - # a2 a1b1 - # `-------> - # a1b1+a2 a1b1 - # v - # a1b1+a2 a1b2 (a1b2 has same content as a2) - # `-------> - # a3b2 a1b2 (autoresolved) - # `-------> - # a3b2 a3b2 - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc') - self.sync(self.db1, self.db2) - for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: - doc = db.get_doc('doc') - doc.set_json(content) - db.put_doc(doc) - self.sync(self.db1, self.db2) - # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict - doc = self.db1.get_doc('doc') - rev1 = doc.rev - self.assertTrue(doc.has_conflicts) - # set db2 to have a doc of {} (same as db1 before the conflict) - doc = self.db2.get_doc('doc') - doc.set_json('{}') - self.db2.put_doc(doc) - rev2 = doc.rev - # sync it across - self.sync(self.db1, self.db2) - # tadaa! - doc = self.db1.get_doc('doc') - self.assertFalse(doc.has_conflicts) - vec1 = vectorclock.VectorClockRev(rev1) - vec2 = vectorclock.VectorClockRev(rev2) - vec3 = vectorclock.VectorClockRev(doc.rev) - self.assertTrue(vec3.is_newer(vec1)) - self.assertTrue(vec3.is_newer(vec2)) - # because the conflict is on the source, sync it another time - self.sync(self.db1, self.db2) - # make sure db2 now has the exact same thing - doc1 = self.db1.get_doc('doc') - self.assertGetEncryptedDoc( - self.db2, - doc1.doc_id, doc1.rev, doc1.get_json(), False) - - def test_sync_autoresolves_moar_backwards(self): - # here we would test that when a database that has a conflicted - # document is the target of a sync, and the source database has a - # revision of the conflicted document that is newer than the target - # database's, and that source's database's document's content is the - # same as the target's document's conflict's, the target's document's - # conflict gets autoresolved, and the document's revision bumped. - # - # Despite that, in Soledad we suppose that the server never syncs, so - # it never has conflicted documents. Also, if it had, convergence - # would not be possible by checking document's contents because they - # would be encrypted in server. - # - # Therefore we suppress this test. - pass - - def test_sync_autoresolves_moar_backwards_three(self): - # here we would test that when a database that has a conflicted - # document is the target of a sync, and the source database has a - # revision of the conflicted document that is newer than the target - # database's, and that source's database's document's content is the - # same as the target's document's conflict's, the target's document's - # conflict gets autoresolved, and the document's revision bumped. - # - # We use the same reasoning from the last test to suppress this one. - pass - - def test_sync_pulling_doesnt_update_other_if_changed(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db2.create_doc_from_json(tests.simple_doc) - # After the local side has sent its list of docs, before we start - # receiving the "targets" response, we update the local database with a - # new record. - # When we finish synchronizing, we can notice that something locally - # was updated, and we cannot tell c2 our new updated generation - - def before_get_docs(state): - if state != 'before get_docs': - return - self.db1.create_doc_from_json(tests.simple_doc) - - self.assertEqual(0, self.sync(self.db1, self.db2, - trace_hook=before_get_docs)) - self.assertLastExchangeLog(self.db2, - {'receive': - {'docs': [], 'last_known_gen': 0}, - 'return': - {'docs': [(doc.doc_id, doc.rev)], - 'last_gen': 1}}) - self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) - # c2 should not have gotten a '_record_sync_info' call, because the - # local database had been updated more than just by the messages - # returned from c2. - self.assertEqual( - (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) - - def test_sync_doesnt_update_other_if_nothing_pulled(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc) - - def no_record_sync_info(state): - if state != 'record_sync_info': - return - self.fail('SyncTarget.record_sync_info was called') - self.assertEqual(1, self.sync(self.db1, self.db2, - trace_hook_shallow=no_record_sync_info)) - self.assertEqual( - 1, - self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)[0]) - - def test_sync_ignores_convergence(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'both') - doc = self.db1.create_doc_from_json(tests.simple_doc) - self.db3 = self.create_database('test3', 'target') - self.assertEqual(1, self.sync(self.db1, self.db3)) - self.assertEqual(0, self.sync(self.db2, self.db3)) - self.assertEqual(1, self.sync(self.db1, self.db2)) - self.assertLastExchangeLog(self.db2, - {'receive': - {'docs': [(doc.doc_id, doc.rev)], - 'source_uid': 'test1', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': {'docs': [], 'last_gen': 1}}) - - def test_sync_ignores_superseded(self): - self.db1 = self.create_database('test1', 'both') - self.db2 = self.create_database('test2', 'both') - doc = self.db1.create_doc_from_json(tests.simple_doc) - doc_rev1 = doc.rev - self.db3 = self.create_database('test3', 'target') - self.sync(self.db1, self.db3) - self.sync(self.db2, self.db3) - new_content = '{"key": "altval"}' - doc.set_json(new_content) - self.db1.put_doc(doc) - doc_rev2 = doc.rev - self.sync(self.db2, self.db1) - self.assertLastExchangeLog(self.db1, - {'receive': - {'docs': [(doc.doc_id, doc_rev1)], - 'source_uid': 'test2', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': - {'docs': [(doc.doc_id, doc_rev2)], - 'last_gen': 2}}) - self.assertGetDoc(self.db1, doc.doc_id, doc_rev2, new_content, False) - - def test_sync_sees_remote_conflicted(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc1 = self.db1.create_doc_from_json(tests.simple_doc) - doc_id = doc1.doc_id - doc1_rev = doc1.rev - self.db1.create_index('test-idx', 'key') - new_doc = '{"key": "altval"}' - doc2 = self.db2.create_doc_from_json(new_doc, doc_id=doc_id) - doc2_rev = doc2.rev - self.assertTransactionLog([doc1.doc_id], self.db1) - self.sync(self.db1, self.db2) - self.assertLastExchangeLog(self.db2, - {'receive': - {'docs': [(doc_id, doc1_rev)], - 'source_uid': 'test1', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': - {'docs': [(doc_id, doc2_rev)], - 'last_gen': 1}}) - self.assertTransactionLog([doc_id, doc_id], self.db1) - self.assertGetDoc(self.db1, doc_id, doc2_rev, new_doc, True) - self.assertGetDoc(self.db2, doc_id, doc2_rev, new_doc, False) - from_idx = self.db1.get_from_index('test-idx', 'altval')[0] - self.assertEqual(doc2.doc_id, from_idx.doc_id) - self.assertEqual(doc2.rev, from_idx.rev) - self.assertTrue(from_idx.has_conflicts) - self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) - - def test_sync_sees_remote_delete_conflicted(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc1 = self.db1.create_doc_from_json(tests.simple_doc) - doc_id = doc1.doc_id - self.db1.create_index('test-idx', 'key') - self.sync(self.db1, self.db2) - doc2 = self.make_document(doc1.doc_id, doc1.rev, doc1.get_json()) - new_doc = '{"key": "altval"}' - doc1.set_json(new_doc) - self.db1.put_doc(doc1) - self.db2.delete_doc(doc2) - self.assertTransactionLog([doc_id, doc_id], self.db1) - self.sync(self.db1, self.db2) - self.assertLastExchangeLog(self.db2, - {'receive': - {'docs': [(doc_id, doc1.rev)], - 'source_uid': 'test1', - 'source_gen': 2, 'last_known_gen': 1}, - 'return': {'docs': [(doc_id, doc2.rev)], - 'last_gen': 2}}) - self.assertTransactionLog([doc_id, doc_id, doc_id], self.db1) - self.assertGetDocIncludeDeleted(self.db1, doc_id, doc2.rev, None, True) - self.assertGetDocIncludeDeleted( - self.db2, doc_id, doc2.rev, None, False) - self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) - - def test_sync_local_race_conflicted(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db1.create_doc_from_json(tests.simple_doc) - doc_id = doc.doc_id - doc1_rev = doc.rev - self.db1.create_index('test-idx', 'key') - self.sync(self.db1, self.db2) - content1 = '{"key": "localval"}' - content2 = '{"key": "altval"}' - doc.set_json(content2) - self.db2.put_doc(doc) - doc2_rev2 = doc.rev - triggered = [] - - def after_whatschanged(state): - if state != 'after whats_changed': - return - triggered.append(True) - doc = self.make_document(doc_id, doc1_rev, content1) - self.db1.put_doc(doc) - - self.sync(self.db1, self.db2, trace_hook=after_whatschanged) - self.assertEqual([True], triggered) - self.assertGetDoc(self.db1, doc_id, doc2_rev2, content2, True) - from_idx = self.db1.get_from_index('test-idx', 'altval')[0] - self.assertEqual(doc.doc_id, from_idx.doc_id) - self.assertEqual(doc.rev, from_idx.rev) - self.assertTrue(from_idx.has_conflicts) - self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) - self.assertEqual([], self.db1.get_from_index('test-idx', 'localval')) - - def test_sync_propagates_deletes(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'both') - doc1 = self.db1.create_doc_from_json(tests.simple_doc) - doc_id = doc1.doc_id - self.db1.create_index('test-idx', 'key') - self.sync(self.db1, self.db2) - self.db2.create_index('test-idx', 'key') - self.db3 = self.create_database('test3', 'target') - self.sync(self.db1, self.db3) - self.db1.delete_doc(doc1) - deleted_rev = doc1.rev - self.sync(self.db1, self.db2) - self.assertLastExchangeLog(self.db2, - {'receive': - {'docs': [(doc_id, deleted_rev)], - 'source_uid': 'test1', - 'source_gen': 2, 'last_known_gen': 1}, - 'return': {'docs': [], 'last_gen': 2}}) - self.assertGetDocIncludeDeleted( - self.db1, doc_id, deleted_rev, None, False) - self.assertGetDocIncludeDeleted( - self.db2, doc_id, deleted_rev, None, False) - self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) - self.assertEqual([], self.db2.get_from_index('test-idx', 'value')) - self.sync(self.db2, self.db3) - self.assertLastExchangeLog(self.db3, - {'receive': - {'docs': [(doc_id, deleted_rev)], - 'source_uid': 'test2', - 'source_gen': 2, - 'last_known_gen': 0}, - 'return': - {'docs': [], 'last_gen': 2}}) - self.assertGetDocIncludeDeleted( - self.db3, doc_id, deleted_rev, None, False) - - def test_sync_propagates_deletes_2(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json('{"a": "1"}', doc_id='the-doc') - self.sync(self.db1, self.db2) - doc1_2 = self.db2.get_doc('the-doc') - self.db2.delete_doc(doc1_2) - self.sync(self.db1, self.db2) - self.assertGetDocIncludeDeleted( - self.db1, 'the-doc', doc1_2.rev, None, False) - - def test_sync_detects_identical_replica_uid(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test1', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') - self.assertRaises( - errors.InvalidReplicaUID, self.sync, self.db1, self.db2) - - def test_optional_sync_preserve_json(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - cont1 = '{ "a": 2 }' - cont2 = '{ "b":3}' - self.db1.create_doc_from_json(cont1, doc_id="1") - self.db2.create_doc_from_json(cont2, doc_id="2") - self.sync(self.db1, self.db2) - self.assertEqual(cont1, self.db2.get_doc("1").get_json()) - self.assertEqual(cont2, self.db1.get_doc("2").get_json()) - - def test_sync_propagates_resolution(self): - """ - Test if synchronization propagates resolution. - - This test was adapted to decrypt remote content before assert. - """ - self.db1 = self.create_database('test1', 'both') - self.db2 = self.create_database('test2', 'both') - doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') - db3 = self.create_database('test3', 'both') - self.sync(self.db2, self.db1) - self.assertEqual( - self.db1._get_generation_info(), - self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)) - self.assertEqual( - self.db2._get_generation_info(), - self.db1._get_replica_gen_and_trans_id(self.db2._replica_uid)) - self.sync(db3, self.db1) - # update on 2 - doc2 = self.make_document('the-doc', doc1.rev, '{"a": 2}') - self.db2.put_doc(doc2) - self.sync(self.db2, db3) - self.assertEqual(db3.get_doc('the-doc').rev, doc2.rev) - # update on 1 - doc1.set_json('{"a": 3}') - self.db1.put_doc(doc1) - # conflicts - self.sync(self.db2, self.db1) - self.sync(db3, self.db1) - self.assertTrue(self.db2.get_doc('the-doc').has_conflicts) - self.assertTrue(db3.get_doc('the-doc').has_conflicts) - # resolve - conflicts = self.db2.get_doc_conflicts('the-doc') - doc4 = self.make_document('the-doc', None, '{"a": 4}') - revs = [doc.rev for doc in conflicts] - self.db2.resolve_doc(doc4, revs) - doc2 = self.db2.get_doc('the-doc') - self.assertEqual(doc4.get_json(), doc2.get_json()) - self.assertFalse(doc2.has_conflicts) - self.sync(self.db2, db3) - doc3 = db3.get_doc('the-doc') - if ENC_SCHEME_KEY in doc3.content: - _crypto = self._soledad._crypto - key = _crypto.doc_passphrase(doc3.doc_id) - secret = _crypto.secret - doc3.set_json(decrypt_doc_dict( - doc3.content, - doc3.doc_id, doc3.rev, key, secret)) - self.assertEqual(doc4.get_json(), doc3.get_json()) - self.assertFalse(doc3.has_conflicts) - self.db1.close() - self.db2.close() - db3.close() - - def test_sync_puts_changes(self): - """ - Test if sync puts changes in remote replica. - - This test was adapted to decrypt remote content before assert. - """ - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db1.create_doc_from_json(tests.simple_doc) - self.assertEqual(1, self.sync(self.db1, self.db2)) - self.assertGetEncryptedDoc( - self.db2, doc.doc_id, doc.rev, tests.simple_doc, False) - self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) - self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [(doc.doc_id, doc.rev)], - 'source_uid': 'test1', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': {'docs': [], 'last_gen': 1}}) - - def test_sync_pulls_changes(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db2.create_doc_from_json(tests.simple_doc) - self.db1.create_index('test-idx', 'key') - self.assertEqual(0, self.sync(self.db1, self.db2)) - self.assertGetDoc(self.db1, doc.doc_id, doc.rev, - tests.simple_doc, False) - self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) - self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) - self.assertLastExchangeLog(self.db2, - {'receive': - {'docs': [], 'last_known_gen': 0}, - 'return': - {'docs': [(doc.doc_id, doc.rev)], - 'last_gen': 1}}) - self.assertEqual([doc], self.db1.get_from_index('test-idx', 'value')) - - def test_sync_supersedes_conflicts(self): - self.db1 = self.create_database('test1', 'both') - self.db2 = self.create_database('test2', 'target') - self.db3 = self.create_database('test3', 'both') - doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') - self.db2.create_doc_from_json('{"b": 1}', doc_id='the-doc') - self.db3.create_doc_from_json('{"c": 1}', doc_id='the-doc') - self.sync(self.db3, self.db1) - self.assertEqual( - self.db1._get_generation_info(), - self.db3._get_replica_gen_and_trans_id(self.db1._replica_uid)) - self.assertEqual( - self.db3._get_generation_info(), - self.db1._get_replica_gen_and_trans_id(self.db3._replica_uid)) - self.sync(self.db3, self.db2) - self.assertEqual( - self.db2._get_generation_info(), - self.db3._get_replica_gen_and_trans_id(self.db2._replica_uid)) - self.assertEqual( - self.db3._get_generation_info(), - self.db2._get_replica_gen_and_trans_id(self.db3._replica_uid)) - self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) - doc1.set_json('{"a": 2}') - self.db1.put_doc(doc1) - self.sync(self.db3, self.db1) - # original doc1 should have been removed from conflicts - self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) - - def test_sync_stops_after_get_sync_info(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc) - self.sync(self.db1, self.db2) - - def put_hook(state): - self.fail("Tracehook triggered for %s" % (state,)) - - self.sync(self.db1, self.db2, trace_hook_shallow=put_hook) - - def test_sync_detects_rollback_in_source(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') - self.sync(self.db1, self.db2) - self.db1_copy = self.copy_database(self.db1) - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.sync(self.db1, self.db2) - self.assertRaises( - errors.InvalidGeneration, self.sync, self.db1_copy, self.db2) - - def test_sync_detects_rollback_in_target(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.db2_copy = self.copy_database(self.db2) - self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.sync(self.db1, self.db2) - self.assertRaises( - errors.InvalidGeneration, self.sync, self.db1, self.db2_copy) - - def test_sync_detects_diverged_source(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db3 = self.copy_database(self.db1) - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.db3.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.assertRaises( - errors.InvalidTransactionId, self.sync, self.db3, self.db2) - - def test_sync_detects_diverged_target(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db3 = self.copy_database(self.db2) - self.db3.create_doc_from_json(tests.nested_doc, doc_id="divergent") - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.assertRaises( - errors.InvalidTransactionId, self.sync, self.db1, self.db3) - - def test_sync_detects_rollback_and_divergence_in_source(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') - self.sync(self.db1, self.db2) - self.db1_copy = self.copy_database(self.db1) - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.sync(self.db1, self.db2) - self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.assertRaises( - errors.InvalidTransactionId, self.sync, self.db1_copy, self.db2) - - def test_sync_detects_rollback_and_divergence_in_target(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.db2_copy = self.copy_database(self.db2) - self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.sync(self.db1, self.db2) - self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.assertRaises( - errors.InvalidTransactionId, self.sync, self.db1, self.db2_copy) - - -def make_local_db_and_soledad_target( - test, path='test', - source_replica_uid=uuid4().hex): - test.startTwistedServer() - replica_uid = os.path.basename(path) - db = test.request_state._create_database(replica_uid) - sync_db = test._soledad._sync_db - sync_enc_pool = test._soledad._sync_enc_pool - st = soledad_sync_target( - test, db._dbname, - source_replica_uid=source_replica_uid, - sync_db=sync_db, - sync_enc_pool=sync_enc_pool) - return db, st - -target_scenarios = [ - ('leap', { - 'create_db_and_target': make_local_db_and_soledad_target, - 'make_app_with_state': make_soledad_app, - 'do_sync': sync_via_synchronizer_and_soledad}), -] diff --git a/testing/tests/sqlcipher/test_async.py b/testing/tests/sqlcipher/test_async.py new file mode 100644 index 00000000..42c315fe --- /dev/null +++ b/testing/tests/sqlcipher/test_async.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- +# test_async.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +import os +import hashlib + +from twisted.internet import defer + +from test_soledad.util import BaseSoledadTest +from leap.soledad.client import adbapi +from leap.soledad.client.sqlcipher import SQLCipherOptions + + +class ASyncSQLCipherRetryTestCase(BaseSoledadTest): + + """ + Test asynchronous SQLCipher operation. + """ + + NUM_DOCS = 5000 + + def setUp(self): + BaseSoledadTest.setUp(self) + self._dbpool = self._get_dbpool() + + def tearDown(self): + self._dbpool.close() + BaseSoledadTest.tearDown(self) + + def _get_dbpool(self): + tmpdb = os.path.join(self.tempdir, "test.soledad") + opts = SQLCipherOptions(tmpdb, "secret", create=True) + return adbapi.getConnectionPool(opts) + + def _get_sample(self): + if not getattr(self, "_sample", None): + dirname = os.path.dirname(os.path.realpath(__file__)) + sample_file = os.path.join(dirname, "hacker_crackdown.txt") + with open(sample_file) as f: + self._sample = f.readlines() + return self._sample + + def test_concurrent_puts_fail_with_few_retries_and_small_timeout(self): + """ + Test if concurrent updates to the database with small timeout and + small number of retries fail with "database is locked" error. + + Many concurrent write attempts to the same sqlcipher database may fail + when the timeout is small and there are no retries. This test will + pass if any of the attempts to write the database fail. + + This test is much dependent on the environment and its result intends + to contrast with the test for the workaround for the "database is + locked" problem, which is addressed by the "test_concurrent_puts" test + below. + + If this test ever fails, it means that either (1) the platform where + you are running is it very powerful and you should try with an even + lower timeout value, or (2) the bug has been solved by a better + implementation of the underlying database pool, and thus this test + should be removed from the test suite. + """ + + old_timeout = adbapi.SQLCIPHER_CONNECTION_TIMEOUT + old_max_retries = adbapi.SQLCIPHER_MAX_RETRIES + + adbapi.SQLCIPHER_CONNECTION_TIMEOUT = 1 + adbapi.SQLCIPHER_MAX_RETRIES = 1 + + def _create_doc(doc): + return self._dbpool.runU1DBQuery("create_doc", doc) + + def _insert_docs(): + deferreds = [] + for i in range(self.NUM_DOCS): + payload = self._get_sample()[i] + chash = hashlib.sha256(payload).hexdigest() + doc = {"number": i, "payload": payload, 'chash': chash} + d = _create_doc(doc) + deferreds.append(d) + return defer.gatherResults(deferreds, consumeErrors=True) + + def _errback(e): + if e.value[0].getErrorMessage() == "database is locked": + adbapi.SQLCIPHER_CONNECTION_TIMEOUT = old_timeout + adbapi.SQLCIPHER_MAX_RETRIES = old_max_retries + return defer.succeed("") + raise Exception + + d = _insert_docs() + d.addCallback(lambda _: self._dbpool.runU1DBQuery("get_all_docs")) + d.addErrback(_errback) + return d + + def test_concurrent_puts(self): + """ + Test that many concurrent puts succeed. + + Currently, there's a known problem with the concurrent database pool + which is that many concurrent attempts to write to the database may + fail when the lock timeout is small and when there are no (or few) + retries. We currently workaround this problem by increasing the + timeout and the number of retries. + + Should this test ever fail, it probably means that the timeout and/or + number of retries should be increased for the platform you're running + the test. If the underlying database pool is ever fixed, then the test + above will fail and we should remove this comment from here. + """ + + def _create_doc(doc): + return self._dbpool.runU1DBQuery("create_doc", doc) + + def _insert_docs(): + deferreds = [] + for i in range(self.NUM_DOCS): + payload = self._get_sample()[i] + chash = hashlib.sha256(payload).hexdigest() + doc = {"number": i, "payload": payload, 'chash': chash} + d = _create_doc(doc) + deferreds.append(d) + return defer.gatherResults(deferreds, consumeErrors=True) + + def _count_docs(results): + _, docs = results + if self.NUM_DOCS == len(docs): + return defer.succeed("") + raise Exception + + d = _insert_docs() + d.addCallback(lambda _: self._dbpool.runU1DBQuery("get_all_docs")) + d.addCallback(_count_docs) + return d diff --git a/testing/tests/sqlcipher/test_sqlcipher.py b/testing/tests/sqlcipher/test_sqlcipher.py new file mode 100644 index 00000000..11472d46 --- /dev/null +++ b/testing/tests/sqlcipher/test_sqlcipher.py @@ -0,0 +1,705 @@ +# -*- coding: utf-8 -*- +# test_sqlcipher.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test sqlcipher backend internals. +""" +import os +import time +import threading +import tempfile +import shutil + +from pysqlcipher import dbapi2 +from testscenarios import TestWithScenarios + +# l2db stuff. +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db import query_parser +from leap.soledad.common.l2db.backends.sqlite_backend \ + import SQLitePartialExpandDatabase + +# soledad stuff. +from leap.soledad.common import soledad_assert +from leap.soledad.common.document import SoledadDocument +from leap.soledad.client.sqlcipher import SQLCipherDatabase +from leap.soledad.client.sqlcipher import SQLCipherOptions +from leap.soledad.client.sqlcipher import DatabaseIsNotEncrypted + +# u1db tests stuff. +from test_soledad import u1db_tests as tests +from test_soledad.u1db_tests import test_backends +from test_soledad.u1db_tests import test_open +from test_soledad.util import SQLCIPHER_SCENARIOS +from test_soledad.util import PASSWORD +from test_soledad.util import BaseSoledadTest + + +def sqlcipher_open(path, passphrase, create=True, document_factory=None): + return SQLCipherDatabase( + SQLCipherOptions(path, passphrase, create=create)) + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_common_backend`. +# ----------------------------------------------------------------------------- + +class TestSQLCipherBackendImpl(tests.TestCase): + + def test__allocate_doc_id(self): + db = sqlcipher_open(':memory:', PASSWORD) + doc_id1 = db._allocate_doc_id() + self.assertTrue(doc_id1.startswith('D-')) + self.assertEqual(34, len(doc_id1)) + int(doc_id1[len('D-'):], 16) + self.assertNotEqual(doc_id1, db._allocate_doc_id()) + db.close() + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_backends`. +# ----------------------------------------------------------------------------- + +class SQLCipherTests(TestWithScenarios, test_backends.AllDatabaseTests): + scenarios = SQLCIPHER_SCENARIOS + + +class SQLCipherDatabaseTests(TestWithScenarios, + test_backends.LocalDatabaseTests): + scenarios = SQLCIPHER_SCENARIOS + + +class SQLCipherValidateGenNTransIdTests( + TestWithScenarios, + test_backends.LocalDatabaseValidateGenNTransIdTests): + scenarios = SQLCIPHER_SCENARIOS + + +class SQLCipherValidateSourceGenTests( + TestWithScenarios, + test_backends.LocalDatabaseValidateSourceGenTests): + scenarios = SQLCIPHER_SCENARIOS + + +class SQLCipherWithConflictsTests( + TestWithScenarios, + test_backends.LocalDatabaseWithConflictsTests): + scenarios = SQLCIPHER_SCENARIOS + + +class SQLCipherIndexTests( + TestWithScenarios, test_backends.DatabaseIndexTests): + scenarios = SQLCIPHER_SCENARIOS + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_sqlite_backend`. +# ----------------------------------------------------------------------------- + +class TestSQLCipherDatabase(tests.TestCase): + """ + Tests from u1db.tests.test_sqlite_backend.TestSQLiteDatabase. + """ + + def test_atomic_initialize(self): + # This test was modified to ensure that db2.close() is called within + # the thread that created the database. + tmpdir = self.createTempDir() + dbname = os.path.join(tmpdir, 'atomic.db') + + t2 = None # will be a thread + + class SQLCipherDatabaseTesting(SQLCipherDatabase): + _index_storage_value = "testing" + + def __init__(self, dbname, ntry): + self._try = ntry + self._is_initialized_invocations = 0 + SQLCipherDatabase.__init__( + self, + SQLCipherOptions(dbname, PASSWORD)) + + def _is_initialized(self, c): + res = \ + SQLCipherDatabase._is_initialized(self, c) + if self._try == 1: + self._is_initialized_invocations += 1 + if self._is_initialized_invocations == 2: + t2.start() + # hard to do better and have a generic test + time.sleep(0.05) + return res + + class SecondTry(threading.Thread): + + outcome2 = [] + + def run(self): + try: + db2 = SQLCipherDatabaseTesting(dbname, 2) + except Exception, e: + SecondTry.outcome2.append(e) + else: + SecondTry.outcome2.append(db2) + + t2 = SecondTry() + db1 = SQLCipherDatabaseTesting(dbname, 1) + t2.join() + + self.assertIsInstance(SecondTry.outcome2[0], SQLCipherDatabaseTesting) + self.assertTrue(db1._is_initialized(db1._get_sqlite_handle().cursor())) + db1.close() + + +class TestSQLCipherPartialExpandDatabase(tests.TestCase): + """ + Tests from u1db.tests.test_sqlite_backend.TestSQLitePartialExpandDatabase. + """ + + # The following tests had to be cloned from u1db because they all + # instantiate the backend directly, so we need to change that in order to + # our backend be instantiated in place. + + def setUp(self): + self.db = sqlcipher_open(':memory:', PASSWORD) + + def tearDown(self): + self.db.close() + + def test_default_replica_uid(self): + self.assertIsNot(None, self.db._replica_uid) + self.assertEqual(32, len(self.db._replica_uid)) + int(self.db._replica_uid, 16) + + def test__parse_index(self): + g = self.db._parse_index_definition('fieldname') + self.assertIsInstance(g, query_parser.ExtractField) + self.assertEqual(['fieldname'], g.field) + + def test__update_indexes(self): + g = self.db._parse_index_definition('fieldname') + c = self.db._get_sqlite_handle().cursor() + self.db._update_indexes('doc-id', {'fieldname': 'val'}, + [('fieldname', g)], c) + c.execute('SELECT doc_id, field_name, value FROM document_fields') + self.assertEqual([('doc-id', 'fieldname', 'val')], + c.fetchall()) + + def test_create_database(self): + raw_db = self.db._get_sqlite_handle() + self.assertNotEqual(None, raw_db) + + def test__set_replica_uid(self): + # Start from scratch, so that replica_uid isn't set. + self.assertIsNot(None, self.db._real_replica_uid) + self.assertIsNot(None, self.db._replica_uid) + self.db._set_replica_uid('foo') + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT value FROM u1db_config WHERE name='replica_uid'") + self.assertEqual(('foo',), c.fetchone()) + self.assertEqual('foo', self.db._real_replica_uid) + self.assertEqual('foo', self.db._replica_uid) + self.db._close_sqlite_handle() + self.assertEqual('foo', self.db._replica_uid) + + def test__open_database(self): + # SQLCipherDatabase has no _open_database() method, so we just pass + # (and test for the same funcionality on test_open_database_existing() + # below). + pass + + def test__open_database_with_factory(self): + # SQLCipherDatabase has no _open_database() method. + pass + + def test__open_database_non_existent(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/non-existent.sqlite' + self.assertRaises(errors.DatabaseDoesNotExist, + sqlcipher_open, + path, PASSWORD, create=False) + + def test__open_database_during_init(self): + # The purpose of this test is to ensure that _open_database() parallel + # db initialization behaviour is correct. As SQLCipherDatabase does + # not have an _open_database() method, we just do not implement this + # test. + pass + + def test__open_database_invalid(self): + # This test was modified to ensure that an empty database file will + # raise a DatabaseIsNotEncrypted exception instead of a + # dbapi2.OperationalError exception. + temp_dir = self.createTempDir(prefix='u1db-test-') + path1 = temp_dir + '/invalid1.db' + with open(path1, 'wb') as f: + f.write("") + self.assertRaises(DatabaseIsNotEncrypted, + sqlcipher_open, path1, + PASSWORD) + with open(path1, 'wb') as f: + f.write("invalid") + self.assertRaises(dbapi2.DatabaseError, + sqlcipher_open, path1, + PASSWORD) + + def test_open_database_existing(self): + # In the context of SQLCipherDatabase, where no _open_database() + # method exists and thus there's no call to _which_index_storage(), + # this test tests for the same functionality as + # test_open_database_create() below. So, we just pass. + pass + + def test_open_database_with_factory(self): + # SQLCipherDatabase's constructor has no factory parameter. + pass + + def test_open_database_create(self): + # SQLCipherDatabas has no open_database() method, so we just test for + # the actual database constructor effects. + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/new.sqlite' + db1 = sqlcipher_open(path, PASSWORD, create=True) + db2 = sqlcipher_open(path, PASSWORD, create=False) + self.assertIsInstance(db2, SQLCipherDatabase) + db1.close() + db2.close() + + def test_create_database_initializes_schema(self): + # This test had to be cloned because our implementation of SQLCipher + # backend is referenced with an index_storage_value that includes the + # word "encrypted". See u1db's sqlite_backend and our + # sqlcipher_backend for reference. + raw_db = self.db._get_sqlite_handle() + c = raw_db.cursor() + c.execute("SELECT * FROM u1db_config") + config = dict([(r[0], r[1]) for r in c.fetchall()]) + replica_uid = self.db._replica_uid + self.assertEqual({'sql_schema': '0', 'replica_uid': replica_uid, + 'index_storage': 'expand referenced encrypted'}, + config) + + def test_store_syncable(self): + doc = self.db.create_doc_from_json(tests.simple_doc) + # assert that docs are syncable by default + self.assertEqual(True, doc.syncable) + # assert that we can store syncable = False + doc.syncable = False + self.db.put_doc(doc) + self.assertEqual(False, self.db.get_doc(doc.doc_id).syncable) + # assert that we can store syncable = True + doc.syncable = True + self.db.put_doc(doc) + self.assertEqual(True, self.db.get_doc(doc.doc_id).syncable) + + def test__close_sqlite_handle(self): + raw_db = self.db._get_sqlite_handle() + self.db._close_sqlite_handle() + self.assertRaises(dbapi2.ProgrammingError, + raw_db.cursor) + + def test__get_generation(self): + self.assertEqual(0, self.db._get_generation()) + + def test__get_generation_info(self): + self.assertEqual((0, ''), self.db._get_generation_info()) + + def test_create_index(self): + self.db.create_index('test-idx', "key") + self.assertEqual([('test-idx', ["key"])], self.db.list_indexes()) + + def test_create_index_multiple_fields(self): + self.db.create_index('test-idx', "key", "key2") + self.assertEqual([('test-idx', ["key", "key2"])], + self.db.list_indexes()) + + def test__get_index_definition(self): + self.db.create_index('test-idx', "key", "key2") + # TODO: How would you test that an index is getting used for an SQL + # request? + self.assertEqual(["key", "key2"], + self.db._get_index_definition('test-idx')) + + def test_list_index_mixed(self): + # Make sure that we properly order the output + c = self.db._get_sqlite_handle().cursor() + # We intentionally insert the data in weird ordering, to make sure the + # query still gets it back correctly. + c.executemany("INSERT INTO index_definitions VALUES (?, ?, ?)", + [('idx-1', 0, 'key10'), + ('idx-2', 2, 'key22'), + ('idx-1', 1, 'key11'), + ('idx-2', 0, 'key20'), + ('idx-2', 1, 'key21')]) + self.assertEqual([('idx-1', ['key10', 'key11']), + ('idx-2', ['key20', 'key21', 'key22'])], + self.db.list_indexes()) + + def test_no_indexes_no_document_fields(self): + self.db.create_doc_from_json( + '{"key1": "val1", "key2": "val2"}') + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([], c.fetchall()) + + def test_create_extracts_fields(self): + doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') + doc2 = self.db.create_doc_from_json('{"key1": "valx", "key2": "valy"}') + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([], c.fetchall()) + self.db.create_index('test', 'key1', 'key2') + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual(sorted( + [(doc1.doc_id, "key1", "val1"), + (doc1.doc_id, "key2", "val2"), + (doc2.doc_id, "key1", "valx"), + (doc2.doc_id, "key2", "valy"), ]), sorted(c.fetchall())) + + def test_put_updates_fields(self): + self.db.create_index('test', 'key1', 'key2') + doc1 = self.db.create_doc_from_json( + '{"key1": "val1", "key2": "val2"}') + doc1.content = {"key1": "val1", "key2": "valy"} + self.db.put_doc(doc1) + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([(doc1.doc_id, "key1", "val1"), + (doc1.doc_id, "key2", "valy"), ], c.fetchall()) + + def test_put_updates_nested_fields(self): + self.db.create_index('test', 'key', 'sub.doc') + doc1 = self.db.create_doc_from_json(tests.nested_doc) + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([(doc1.doc_id, "key", "value"), + (doc1.doc_id, "sub.doc", "underneath"), ], + c.fetchall()) + + def test__ensure_schema_rollback(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/rollback.db' + + class SQLitePartialExpandDbTesting(SQLCipherDatabase): + + def _set_replica_uid_in_transaction(self, uid): + super(SQLitePartialExpandDbTesting, + self)._set_replica_uid_in_transaction(uid) + if fail: + raise Exception() + + db = SQLitePartialExpandDbTesting.__new__(SQLitePartialExpandDbTesting) + db._db_handle = dbapi2.connect(path) # db is there but not yet init-ed + fail = True + self.assertRaises(Exception, db._ensure_schema) + fail = False + db._initialize(db._db_handle.cursor()) + + def test_open_database_non_existent(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/non-existent.sqlite' + self.assertRaises(errors.DatabaseDoesNotExist, + sqlcipher_open, path, "123", + create=False) + + def test_delete_database_existent(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/new.sqlite' + db = sqlcipher_open(path, "123", create=True) + db.close() + SQLCipherDatabase.delete_database(path) + self.assertRaises(errors.DatabaseDoesNotExist, + sqlcipher_open, path, "123", + create=False) + + def test_delete_database_nonexistent(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/non-existent.sqlite' + self.assertRaises(errors.DatabaseDoesNotExist, + SQLCipherDatabase.delete_database, path) + + def test__get_indexed_fields(self): + self.db.create_index('idx1', 'a', 'b') + self.assertEqual(set(['a', 'b']), self.db._get_indexed_fields()) + self.db.create_index('idx2', 'b', 'c') + self.assertEqual(set(['a', 'b', 'c']), self.db._get_indexed_fields()) + + def test_indexed_fields_expanded(self): + self.db.create_index('idx1', 'key1') + doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') + self.assertEqual(set(['key1']), self.db._get_indexed_fields()) + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([(doc1.doc_id, 'key1', 'val1')], c.fetchall()) + + def test_create_index_updates_fields(self): + doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') + self.db.create_index('idx1', 'key1') + self.assertEqual(set(['key1']), self.db._get_indexed_fields()) + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([(doc1.doc_id, 'key1', 'val1')], c.fetchall()) + + def assertFormatQueryEquals(self, exp_statement, exp_args, definition, + values): + statement, args = self.db._format_query(definition, values) + self.assertEqual(exp_statement, statement) + self.assertEqual(exp_args, args) + + def test__format_query(self): + self.assertFormatQueryEquals( + "SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM " + "document d, document_fields d0 LEFT OUTER JOIN conflicts c ON " + "c.doc_id = d.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name " + "= ? AND d0.value = ? GROUP BY d.doc_id, d.doc_rev, d.content " + "ORDER BY d0.value;", ["key1", "a"], + ["key1"], ["a"]) + + def test__format_query2(self): + self.assertFormatQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value = ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value = ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value = ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' + 'd0.value, d1.value, d2.value;', + ["key1", "a", "key2", "b", "key3", "c"], + ["key1", "key2", "key3"], ["a", "b", "c"]) + + def test__format_query_wildcard(self): + self.assertFormatQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value = ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value GLOB ? AND d.doc_id = d2.doc_id AND d2.field_name = ? ' + 'AND d2.value NOT NULL GROUP BY d.doc_id, d.doc_rev, d.content ' + 'ORDER BY d0.value, d1.value, d2.value;', + ["key1", "a", "key2", "b*", "key3"], ["key1", "key2", "key3"], + ["a", "b*", "*"]) + + def assertFormatRangeQueryEquals(self, exp_statement, exp_args, definition, + start_value, end_value): + statement, args = self.db._format_range_query( + definition, start_value, end_value) + self.assertEqual(exp_statement, statement) + self.assertEqual(exp_args, args) + + def test__format_range_query(self): + self.assertFormatRangeQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value >= ? AND d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value <= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value <= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' + 'd0.value, d1.value, d2.value;', + ['key1', 'a', 'key2', 'b', 'key3', 'c', 'key1', 'p', 'key2', 'q', + 'key3', 'r'], + ["key1", "key2", "key3"], ["a", "b", "c"], ["p", "q", "r"]) + + def test__format_range_query_no_start(self): + self.assertFormatRangeQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value <= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value <= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' + 'd0.value, d1.value, d2.value;', + ['key1', 'a', 'key2', 'b', 'key3', 'c'], + ["key1", "key2", "key3"], None, ["a", "b", "c"]) + + def test__format_range_query_no_end(self): + self.assertFormatRangeQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value >= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' + 'd0.value, d1.value, d2.value;', + ['key1', 'a', 'key2', 'b', 'key3', 'c'], + ["key1", "key2", "key3"], ["a", "b", "c"], None) + + def test__format_range_query_wildcard(self): + self.assertFormatRangeQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value NOT NULL AND d.doc_id = d0.doc_id AND d0.field_name = ? ' + 'AND d0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? ' + 'AND (d1.value < ? OR d1.value GLOB ?) AND d.doc_id = d2.doc_id ' + 'AND d2.field_name = ? AND d2.value NOT NULL GROUP BY d.doc_id, ' + 'd.doc_rev, d.content ORDER BY d0.value, d1.value, d2.value;', + ['key1', 'a', 'key2', 'b', 'key3', 'key1', 'p', 'key2', 'q', 'q*', + 'key3'], + ["key1", "key2", "key3"], ["a", "b*", "*"], ["p", "q*", "*"]) + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_open`. +# ----------------------------------------------------------------------------- + + +class SQLCipherOpen(test_open.TestU1DBOpen): + + def test_open_no_create(self): + self.assertRaises(errors.DatabaseDoesNotExist, + sqlcipher_open, self.db_path, + PASSWORD, + create=False) + self.assertFalse(os.path.exists(self.db_path)) + + def test_open_create(self): + db = sqlcipher_open(self.db_path, PASSWORD, create=True) + self.addCleanup(db.close) + self.assertTrue(os.path.exists(self.db_path)) + self.assertIsInstance(db, SQLCipherDatabase) + + def test_open_with_factory(self): + db = sqlcipher_open(self.db_path, PASSWORD, create=True, + document_factory=SoledadDocument) + self.addCleanup(db.close) + doc = db.create_doc({}) + self.assertTrue(isinstance(doc, SoledadDocument)) + + def test_open_existing(self): + db = sqlcipher_open(self.db_path, PASSWORD) + self.addCleanup(db.close) + doc = db.create_doc_from_json(tests.simple_doc) + # Even though create=True, we shouldn't wipe the db + db2 = sqlcipher_open(self.db_path, PASSWORD, create=True) + self.addCleanup(db2.close) + doc2 = db2.get_doc(doc.doc_id) + self.assertEqual(doc, doc2) + + def test_open_existing_no_create(self): + db = sqlcipher_open(self.db_path, PASSWORD) + self.addCleanup(db.close) + db2 = sqlcipher_open(self.db_path, PASSWORD, create=False) + self.addCleanup(db2.close) + self.assertIsInstance(db2, SQLCipherDatabase) + + +# ----------------------------------------------------------------------------- +# Tests for actual encryption of the database +# ----------------------------------------------------------------------------- + +class SQLCipherEncryptionTests(BaseSoledadTest): + + """ + Tests to guarantee SQLCipher is indeed encrypting data when storing. + """ + + def _delete_dbfiles(self): + for dbfile in [self.DB_FILE]: + if os.path.exists(dbfile): + os.unlink(dbfile) + + def setUp(self): + # the following come from BaseLeapTest.setUpClass, because + # twisted.trial doesn't support such class methods for setting up + # test classes. + self.old_path = os.environ['PATH'] + self.old_home = os.environ['HOME'] + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + self.home = self.tempdir + bin_tdir = os.path.join( + self.tempdir, + 'bin') + os.environ["PATH"] = bin_tdir + os.environ["HOME"] = self.tempdir + # this is our own stuff + self.DB_FILE = os.path.join(self.tempdir, 'test.db') + self._delete_dbfiles() + + def tearDown(self): + self._delete_dbfiles() + # the following come from BaseLeapTest.tearDownClass, because + # twisted.trial doesn't support such class methods for tearing down + # test classes. + os.environ["PATH"] = self.old_path + os.environ["HOME"] = self.old_home + # safety check! please do not wipe my home... + # XXX needs to adapt to non-linuces + soledad_assert( + self.tempdir.startswith('/tmp/leap_tests-') or + self.tempdir.startswith('/var/folder'), + "beware! tried to remove a dir which does not " + "live in temporal folder!") + shutil.rmtree(self.tempdir) + + def test_try_to_open_encrypted_db_with_sqlite_backend(self): + """ + SQLite backend should not succeed to open SQLCipher databases. + """ + db = sqlcipher_open(self.DB_FILE, PASSWORD) + doc = db.create_doc_from_json(tests.simple_doc) + db.close() + try: + # trying to open an encrypted database with the regular u1db + # backend should raise a DatabaseError exception. + SQLitePartialExpandDatabase(self.DB_FILE, + document_factory=SoledadDocument) + raise DatabaseIsNotEncrypted() + except dbapi2.DatabaseError: + # at this point we know that the regular U1DB sqlcipher backend + # did not succeed on opening the database, so it was indeed + # encrypted. + db = sqlcipher_open(self.DB_FILE, PASSWORD) + doc = db.get_doc(doc.doc_id) + self.assertEqual(tests.simple_doc, doc.get_json(), + 'decrypted content mismatch') + db.close() + + def test_try_to_open_raw_db_with_sqlcipher_backend(self): + """ + SQLCipher backend should not succeed to open unencrypted databases. + """ + db = SQLitePartialExpandDatabase(self.DB_FILE, + document_factory=SoledadDocument) + db.create_doc_from_json(tests.simple_doc) + db.close() + try: + # trying to open the a non-encrypted database with sqlcipher + # backend should raise a DatabaseIsNotEncrypted exception. + db = sqlcipher_open(self.DB_FILE, PASSWORD) + db.close() + raise dbapi2.DatabaseError( + "SQLCipher backend should not be able to open non-encrypted " + "dbs.") + except DatabaseIsNotEncrypted: + pass diff --git a/testing/tests/sync/test_sqlcipher_sync.py b/testing/tests/sync/test_sqlcipher_sync.py new file mode 100644 index 00000000..3cbefc8b --- /dev/null +++ b/testing/tests/sync/test_sqlcipher_sync.py @@ -0,0 +1,730 @@ +# -*- coding: utf-8 -*- +# test_sqlcipher.py +# Copyright (C) 2013-2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test sqlcipher backend sync. +""" +import os + +from uuid import uuid4 + +from testscenarios import TestWithScenarios + +from leap.soledad.common.l2db import sync +from leap.soledad.common.l2db import vectorclock +from leap.soledad.common.l2db import errors + +from leap.soledad.common.crypto import ENC_SCHEME_KEY +from leap.soledad.client.crypto import decrypt_doc_dict +from leap.soledad.client.http_target import SoledadHTTPSyncTarget + +from test_soledad import u1db_tests as tests +from test_soledad.util import SQLCIPHER_SCENARIOS +from test_soledad.util import make_soledad_app +from test_soledad.util import soledad_sync_target +from test_soledad.util import BaseSoledadTest + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_sync`. +# ----------------------------------------------------------------------------- + +def sync_via_synchronizer_and_soledad(test, db_source, db_target, + trace_hook=None, + trace_hook_shallow=None): + if trace_hook: + test.skipTest("full trace hook unsupported over http") + path = test._http_at[db_target] + target = SoledadHTTPSyncTarget.connect( + test.getURL(path), test._soledad._crypto) + target.set_token_credentials('user-uuid', 'auth-token') + if trace_hook_shallow: + target._set_trace_hook_shallow(trace_hook_shallow) + return sync.Synchronizer(db_source, target).sync() + + +def sync_via_synchronizer(test, db_source, db_target, + trace_hook=None, + trace_hook_shallow=None): + target = db_target.get_sync_target() + trace_hook = trace_hook or trace_hook_shallow + if trace_hook: + target._set_trace_hook(trace_hook) + return sync.Synchronizer(db_source, target).sync() + + +sync_scenarios = [] +for name, scenario in SQLCIPHER_SCENARIOS: + scenario['do_sync'] = sync_via_synchronizer + sync_scenarios.append((name, scenario)) + + +class SQLCipherDatabaseSyncTests( + TestWithScenarios, + tests.DatabaseBaseTests, + BaseSoledadTest): + + """ + Test for succesfull sync between SQLCipher and LeapBackend. + + Some of the tests in this class had to be adapted because the remote + backend always receive encrypted content, and so it can not rely on + document's content comparison to try to autoresolve conflicts. + """ + + scenarios = sync_scenarios + + def setUp(self): + self._use_tracking = {} + super(tests.DatabaseBaseTests, self).setUp() + + def create_database(self, replica_uid, sync_role=None): + if replica_uid == 'test' and sync_role is None: + # created up the chain by base class but unused + return None + db = self.create_database_for_role(replica_uid, sync_role) + if sync_role: + self._use_tracking[db] = (replica_uid, sync_role) + self.addCleanup(db.close) + return db + + def create_database_for_role(self, replica_uid, sync_role): + # hook point for reuse + return tests.DatabaseBaseTests.create_database(self, replica_uid) + + def sync(self, db_from, db_to, trace_hook=None, + trace_hook_shallow=None): + from_name, from_sync_role = self._use_tracking[db_from] + to_name, to_sync_role = self._use_tracking[db_to] + if from_sync_role not in ('source', 'both'): + raise Exception("%s marked for %s use but used as source" % + (from_name, from_sync_role)) + if to_sync_role not in ('target', 'both'): + raise Exception("%s marked for %s use but used as target" % + (to_name, to_sync_role)) + return self.do_sync(self, db_from, db_to, trace_hook, + trace_hook_shallow) + + def assertLastExchangeLog(self, db, expected): + log = getattr(db, '_last_exchange_log', None) + if log is None: + return + self.assertEqual(expected, log) + + def copy_database(self, db, sync_role=None): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES + # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST + # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS + # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND + # NINJA TO YOUR HOUSE. + db_copy = tests.DatabaseBaseTests.copy_database(self, db) + name, orig_sync_role = self._use_tracking[db] + self._use_tracking[db_copy] = (name + '(copy)', sync_role or + orig_sync_role) + return db_copy + + def test_sync_tracks_db_generation_of_other(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.assertEqual(0, self.sync(self.db1, self.db2)) + self.assertEqual( + (0, ''), self.db1._get_replica_gen_and_trans_id('test2')) + self.assertEqual( + (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) + self.assertLastExchangeLog(self.db2, + {'receive': + {'docs': [], 'last_known_gen': 0}, + 'return': + {'docs': [], 'last_gen': 0}}) + + def test_sync_autoresolves(self): + """ + Test for sync autoresolve remote. + + This test was adapted because the remote database receives encrypted + content and so it can't compare documents contents to autoresolve. + """ + # The remote database can't autoresolve conflicts based on magic + # content convergence, so we modify this test to leave the possibility + # of the remode document ending up in conflicted state. + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc') + rev1 = doc1.rev + doc2 = self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc') + rev2 = doc2.rev + self.sync(self.db1, self.db2) + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + # if remote content is in conflicted state, then document revisions + # will be different. + # self.assertEqual(doc.rev, self.db2.get_doc('doc').rev) + v = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev1))) + self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev2))) + + def test_sync_autoresolves_moar(self): + """ + Test for sync autoresolve local. + + This test was adapted to decrypt remote content before assert. + """ + # here we test that when a database that has a conflicted document is + # the source of a sync, and the target database has a revision of the + # conflicted document that is newer than the source database's, and + # that target's database's document's content is the same as the + # source's document's conflict's, the source's document's conflict gets + # autoresolved, and the source's document's revision bumped. + # + # idea is as follows: + # A B + # a1 - + # `-------> + # a1 a1 + # v v + # a2 a1b1 + # `-------> + # a1b1+a2 a1b1 + # v + # a1b1+a2 a1b2 (a1b2 has same content as a2) + # `-------> + # a3b2 a1b2 (autoresolved) + # `-------> + # a3b2 a3b2 + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc') + self.sync(self.db1, self.db2) + for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: + doc = db.get_doc('doc') + doc.set_json(content) + db.put_doc(doc) + self.sync(self.db1, self.db2) + # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict + doc = self.db1.get_doc('doc') + rev1 = doc.rev + self.assertTrue(doc.has_conflicts) + # set db2 to have a doc of {} (same as db1 before the conflict) + doc = self.db2.get_doc('doc') + doc.set_json('{}') + self.db2.put_doc(doc) + rev2 = doc.rev + # sync it across + self.sync(self.db1, self.db2) + # tadaa! + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + vec1 = vectorclock.VectorClockRev(rev1) + vec2 = vectorclock.VectorClockRev(rev2) + vec3 = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(vec3.is_newer(vec1)) + self.assertTrue(vec3.is_newer(vec2)) + # because the conflict is on the source, sync it another time + self.sync(self.db1, self.db2) + # make sure db2 now has the exact same thing + doc1 = self.db1.get_doc('doc') + self.assertGetEncryptedDoc( + self.db2, + doc1.doc_id, doc1.rev, doc1.get_json(), False) + + def test_sync_autoresolves_moar_backwards(self): + # here we would test that when a database that has a conflicted + # document is the target of a sync, and the source database has a + # revision of the conflicted document that is newer than the target + # database's, and that source's database's document's content is the + # same as the target's document's conflict's, the target's document's + # conflict gets autoresolved, and the document's revision bumped. + # + # Despite that, in Soledad we suppose that the server never syncs, so + # it never has conflicted documents. Also, if it had, convergence + # would not be possible by checking document's contents because they + # would be encrypted in server. + # + # Therefore we suppress this test. + pass + + def test_sync_autoresolves_moar_backwards_three(self): + # here we would test that when a database that has a conflicted + # document is the target of a sync, and the source database has a + # revision of the conflicted document that is newer than the target + # database's, and that source's database's document's content is the + # same as the target's document's conflict's, the target's document's + # conflict gets autoresolved, and the document's revision bumped. + # + # We use the same reasoning from the last test to suppress this one. + pass + + def test_sync_pulling_doesnt_update_other_if_changed(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db2.create_doc_from_json(tests.simple_doc) + # After the local side has sent its list of docs, before we start + # receiving the "targets" response, we update the local database with a + # new record. + # When we finish synchronizing, we can notice that something locally + # was updated, and we cannot tell c2 our new updated generation + + def before_get_docs(state): + if state != 'before get_docs': + return + self.db1.create_doc_from_json(tests.simple_doc) + + self.assertEqual(0, self.sync(self.db1, self.db2, + trace_hook=before_get_docs)) + self.assertLastExchangeLog(self.db2, + {'receive': + {'docs': [], 'last_known_gen': 0}, + 'return': + {'docs': [(doc.doc_id, doc.rev)], + 'last_gen': 1}}) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + # c2 should not have gotten a '_record_sync_info' call, because the + # local database had been updated more than just by the messages + # returned from c2. + self.assertEqual( + (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) + + def test_sync_doesnt_update_other_if_nothing_pulled(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc) + + def no_record_sync_info(state): + if state != 'record_sync_info': + return + self.fail('SyncTarget.record_sync_info was called') + self.assertEqual(1, self.sync(self.db1, self.db2, + trace_hook_shallow=no_record_sync_info)) + self.assertEqual( + 1, + self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)[0]) + + def test_sync_ignores_convergence(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'both') + doc = self.db1.create_doc_from_json(tests.simple_doc) + self.db3 = self.create_database('test3', 'target') + self.assertEqual(1, self.sync(self.db1, self.db3)) + self.assertEqual(0, self.sync(self.db2, self.db3)) + self.assertEqual(1, self.sync(self.db1, self.db2)) + self.assertLastExchangeLog(self.db2, + {'receive': + {'docs': [(doc.doc_id, doc.rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 1}}) + + def test_sync_ignores_superseded(self): + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'both') + doc = self.db1.create_doc_from_json(tests.simple_doc) + doc_rev1 = doc.rev + self.db3 = self.create_database('test3', 'target') + self.sync(self.db1, self.db3) + self.sync(self.db2, self.db3) + new_content = '{"key": "altval"}' + doc.set_json(new_content) + self.db1.put_doc(doc) + doc_rev2 = doc.rev + self.sync(self.db2, self.db1) + self.assertLastExchangeLog(self.db1, + {'receive': + {'docs': [(doc.doc_id, doc_rev1)], + 'source_uid': 'test2', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': + {'docs': [(doc.doc_id, doc_rev2)], + 'last_gen': 2}}) + self.assertGetDoc(self.db1, doc.doc_id, doc_rev2, new_content, False) + + def test_sync_sees_remote_conflicted(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(tests.simple_doc) + doc_id = doc1.doc_id + doc1_rev = doc1.rev + self.db1.create_index('test-idx', 'key') + new_doc = '{"key": "altval"}' + doc2 = self.db2.create_doc_from_json(new_doc, doc_id=doc_id) + doc2_rev = doc2.rev + self.assertTransactionLog([doc1.doc_id], self.db1) + self.sync(self.db1, self.db2) + self.assertLastExchangeLog(self.db2, + {'receive': + {'docs': [(doc_id, doc1_rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': + {'docs': [(doc_id, doc2_rev)], + 'last_gen': 1}}) + self.assertTransactionLog([doc_id, doc_id], self.db1) + self.assertGetDoc(self.db1, doc_id, doc2_rev, new_doc, True) + self.assertGetDoc(self.db2, doc_id, doc2_rev, new_doc, False) + from_idx = self.db1.get_from_index('test-idx', 'altval')[0] + self.assertEqual(doc2.doc_id, from_idx.doc_id) + self.assertEqual(doc2.rev, from_idx.rev) + self.assertTrue(from_idx.has_conflicts) + self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) + + def test_sync_sees_remote_delete_conflicted(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(tests.simple_doc) + doc_id = doc1.doc_id + self.db1.create_index('test-idx', 'key') + self.sync(self.db1, self.db2) + doc2 = self.make_document(doc1.doc_id, doc1.rev, doc1.get_json()) + new_doc = '{"key": "altval"}' + doc1.set_json(new_doc) + self.db1.put_doc(doc1) + self.db2.delete_doc(doc2) + self.assertTransactionLog([doc_id, doc_id], self.db1) + self.sync(self.db1, self.db2) + self.assertLastExchangeLog(self.db2, + {'receive': + {'docs': [(doc_id, doc1.rev)], + 'source_uid': 'test1', + 'source_gen': 2, 'last_known_gen': 1}, + 'return': {'docs': [(doc_id, doc2.rev)], + 'last_gen': 2}}) + self.assertTransactionLog([doc_id, doc_id, doc_id], self.db1) + self.assertGetDocIncludeDeleted(self.db1, doc_id, doc2.rev, None, True) + self.assertGetDocIncludeDeleted( + self.db2, doc_id, doc2.rev, None, False) + self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) + + def test_sync_local_race_conflicted(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db1.create_doc_from_json(tests.simple_doc) + doc_id = doc.doc_id + doc1_rev = doc.rev + self.db1.create_index('test-idx', 'key') + self.sync(self.db1, self.db2) + content1 = '{"key": "localval"}' + content2 = '{"key": "altval"}' + doc.set_json(content2) + self.db2.put_doc(doc) + doc2_rev2 = doc.rev + triggered = [] + + def after_whatschanged(state): + if state != 'after whats_changed': + return + triggered.append(True) + doc = self.make_document(doc_id, doc1_rev, content1) + self.db1.put_doc(doc) + + self.sync(self.db1, self.db2, trace_hook=after_whatschanged) + self.assertEqual([True], triggered) + self.assertGetDoc(self.db1, doc_id, doc2_rev2, content2, True) + from_idx = self.db1.get_from_index('test-idx', 'altval')[0] + self.assertEqual(doc.doc_id, from_idx.doc_id) + self.assertEqual(doc.rev, from_idx.rev) + self.assertTrue(from_idx.has_conflicts) + self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) + self.assertEqual([], self.db1.get_from_index('test-idx', 'localval')) + + def test_sync_propagates_deletes(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'both') + doc1 = self.db1.create_doc_from_json(tests.simple_doc) + doc_id = doc1.doc_id + self.db1.create_index('test-idx', 'key') + self.sync(self.db1, self.db2) + self.db2.create_index('test-idx', 'key') + self.db3 = self.create_database('test3', 'target') + self.sync(self.db1, self.db3) + self.db1.delete_doc(doc1) + deleted_rev = doc1.rev + self.sync(self.db1, self.db2) + self.assertLastExchangeLog(self.db2, + {'receive': + {'docs': [(doc_id, deleted_rev)], + 'source_uid': 'test1', + 'source_gen': 2, 'last_known_gen': 1}, + 'return': {'docs': [], 'last_gen': 2}}) + self.assertGetDocIncludeDeleted( + self.db1, doc_id, deleted_rev, None, False) + self.assertGetDocIncludeDeleted( + self.db2, doc_id, deleted_rev, None, False) + self.assertEqual([], self.db1.get_from_index('test-idx', 'value')) + self.assertEqual([], self.db2.get_from_index('test-idx', 'value')) + self.sync(self.db2, self.db3) + self.assertLastExchangeLog(self.db3, + {'receive': + {'docs': [(doc_id, deleted_rev)], + 'source_uid': 'test2', + 'source_gen': 2, + 'last_known_gen': 0}, + 'return': + {'docs': [], 'last_gen': 2}}) + self.assertGetDocIncludeDeleted( + self.db3, doc_id, deleted_rev, None, False) + + def test_sync_propagates_deletes_2(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json('{"a": "1"}', doc_id='the-doc') + self.sync(self.db1, self.db2) + doc1_2 = self.db2.get_doc('the-doc') + self.db2.delete_doc(doc1_2) + self.sync(self.db1, self.db2) + self.assertGetDocIncludeDeleted( + self.db1, 'the-doc', doc1_2.rev, None, False) + + def test_sync_detects_identical_replica_uid(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test1', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') + self.assertRaises( + errors.InvalidReplicaUID, self.sync, self.db1, self.db2) + + def test_optional_sync_preserve_json(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + cont1 = '{ "a": 2 }' + cont2 = '{ "b":3}' + self.db1.create_doc_from_json(cont1, doc_id="1") + self.db2.create_doc_from_json(cont2, doc_id="2") + self.sync(self.db1, self.db2) + self.assertEqual(cont1, self.db2.get_doc("1").get_json()) + self.assertEqual(cont2, self.db1.get_doc("2").get_json()) + + def test_sync_propagates_resolution(self): + """ + Test if synchronization propagates resolution. + + This test was adapted to decrypt remote content before assert. + """ + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'both') + doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') + db3 = self.create_database('test3', 'both') + self.sync(self.db2, self.db1) + self.assertEqual( + self.db1._get_generation_info(), + self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)) + self.assertEqual( + self.db2._get_generation_info(), + self.db1._get_replica_gen_and_trans_id(self.db2._replica_uid)) + self.sync(db3, self.db1) + # update on 2 + doc2 = self.make_document('the-doc', doc1.rev, '{"a": 2}') + self.db2.put_doc(doc2) + self.sync(self.db2, db3) + self.assertEqual(db3.get_doc('the-doc').rev, doc2.rev) + # update on 1 + doc1.set_json('{"a": 3}') + self.db1.put_doc(doc1) + # conflicts + self.sync(self.db2, self.db1) + self.sync(db3, self.db1) + self.assertTrue(self.db2.get_doc('the-doc').has_conflicts) + self.assertTrue(db3.get_doc('the-doc').has_conflicts) + # resolve + conflicts = self.db2.get_doc_conflicts('the-doc') + doc4 = self.make_document('the-doc', None, '{"a": 4}') + revs = [doc.rev for doc in conflicts] + self.db2.resolve_doc(doc4, revs) + doc2 = self.db2.get_doc('the-doc') + self.assertEqual(doc4.get_json(), doc2.get_json()) + self.assertFalse(doc2.has_conflicts) + self.sync(self.db2, db3) + doc3 = db3.get_doc('the-doc') + if ENC_SCHEME_KEY in doc3.content: + _crypto = self._soledad._crypto + key = _crypto.doc_passphrase(doc3.doc_id) + secret = _crypto.secret + doc3.set_json(decrypt_doc_dict( + doc3.content, + doc3.doc_id, doc3.rev, key, secret)) + self.assertEqual(doc4.get_json(), doc3.get_json()) + self.assertFalse(doc3.has_conflicts) + self.db1.close() + self.db2.close() + db3.close() + + def test_sync_puts_changes(self): + """ + Test if sync puts changes in remote replica. + + This test was adapted to decrypt remote content before assert. + """ + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db1.create_doc_from_json(tests.simple_doc) + self.assertEqual(1, self.sync(self.db1, self.db2)) + self.assertGetEncryptedDoc( + self.db2, doc.doc_id, doc.rev, tests.simple_doc, False) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [(doc.doc_id, doc.rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 1}}) + + def test_sync_pulls_changes(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db2.create_doc_from_json(tests.simple_doc) + self.db1.create_index('test-idx', 'key') + self.assertEqual(0, self.sync(self.db1, self.db2)) + self.assertGetDoc(self.db1, doc.doc_id, doc.rev, + tests.simple_doc, False) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) + self.assertLastExchangeLog(self.db2, + {'receive': + {'docs': [], 'last_known_gen': 0}, + 'return': + {'docs': [(doc.doc_id, doc.rev)], + 'last_gen': 1}}) + self.assertEqual([doc], self.db1.get_from_index('test-idx', 'value')) + + def test_sync_supersedes_conflicts(self): + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'target') + self.db3 = self.create_database('test3', 'both') + doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') + self.db2.create_doc_from_json('{"b": 1}', doc_id='the-doc') + self.db3.create_doc_from_json('{"c": 1}', doc_id='the-doc') + self.sync(self.db3, self.db1) + self.assertEqual( + self.db1._get_generation_info(), + self.db3._get_replica_gen_and_trans_id(self.db1._replica_uid)) + self.assertEqual( + self.db3._get_generation_info(), + self.db1._get_replica_gen_and_trans_id(self.db3._replica_uid)) + self.sync(self.db3, self.db2) + self.assertEqual( + self.db2._get_generation_info(), + self.db3._get_replica_gen_and_trans_id(self.db2._replica_uid)) + self.assertEqual( + self.db3._get_generation_info(), + self.db2._get_replica_gen_and_trans_id(self.db3._replica_uid)) + self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) + doc1.set_json('{"a": 2}') + self.db1.put_doc(doc1) + self.sync(self.db3, self.db1) + # original doc1 should have been removed from conflicts + self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) + + def test_sync_stops_after_get_sync_info(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc) + self.sync(self.db1, self.db2) + + def put_hook(state): + self.fail("Tracehook triggered for %s" % (state,)) + + self.sync(self.db1, self.db2, trace_hook_shallow=put_hook) + + def test_sync_detects_rollback_in_source(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') + self.sync(self.db1, self.db2) + self.db1_copy = self.copy_database(self.db1) + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.sync(self.db1, self.db2) + self.assertRaises( + errors.InvalidGeneration, self.sync, self.db1_copy, self.db2) + + def test_sync_detects_rollback_in_target(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.db2_copy = self.copy_database(self.db2) + self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.sync(self.db1, self.db2) + self.assertRaises( + errors.InvalidGeneration, self.sync, self.db1, self.db2_copy) + + def test_sync_detects_diverged_source(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db3 = self.copy_database(self.db1) + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.db3.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.assertRaises( + errors.InvalidTransactionId, self.sync, self.db3, self.db2) + + def test_sync_detects_diverged_target(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db3 = self.copy_database(self.db2) + self.db3.create_doc_from_json(tests.nested_doc, doc_id="divergent") + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.assertRaises( + errors.InvalidTransactionId, self.sync, self.db1, self.db3) + + def test_sync_detects_rollback_and_divergence_in_source(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') + self.sync(self.db1, self.db2) + self.db1_copy = self.copy_database(self.db1) + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.sync(self.db1, self.db2) + self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.assertRaises( + errors.InvalidTransactionId, self.sync, self.db1_copy, self.db2) + + def test_sync_detects_rollback_and_divergence_in_target(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.db2_copy = self.copy_database(self.db2) + self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.sync(self.db1, self.db2) + self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.assertRaises( + errors.InvalidTransactionId, self.sync, self.db1, self.db2_copy) + + +def make_local_db_and_soledad_target( + test, path='test', + source_replica_uid=uuid4().hex): + test.startTwistedServer() + replica_uid = os.path.basename(path) + db = test.request_state._create_database(replica_uid) + sync_db = test._soledad._sync_db + sync_enc_pool = test._soledad._sync_enc_pool + st = soledad_sync_target( + test, db._dbname, + source_replica_uid=source_replica_uid, + sync_db=sync_db, + sync_enc_pool=sync_enc_pool) + return db, st + +target_scenarios = [ + ('leap', { + 'create_db_and_target': make_local_db_and_soledad_target, + 'make_app_with_state': make_soledad_app, + 'do_sync': sync_via_synchronizer_and_soledad}), +] -- cgit v1.2.3 From 769ce3d7a014b1e1e0fba23df6993271cfd647ba Mon Sep 17 00:00:00 2001 From: drebs Date: Sun, 10 Jul 2016 11:08:18 +0200 Subject: [test] refactor test files --- testing/tests/client/hacker_crackdown.txt | 13005 ------------------- testing/tests/couch/common.py | 81 + testing/tests/couch/test_atomicity.py | 371 + testing/tests/couch/test_backend.py | 115 + testing/tests/couch/test_command.py | 28 + testing/tests/couch/test_couch.py | 1442 -- .../tests/couch/test_couch_operations_atomicity.py | 371 - testing/tests/couch/test_ddocs.py | 209 + testing/tests/couch/test_sync.py | 700 + testing/tests/couch/test_sync_target.py | 343 + testing/tests/sqlcipher/hacker_crackdown.txt | 13005 +++++++++++++++++++ testing/tests/sqlcipher/test_backend.py | 705 + testing/tests/sqlcipher/test_sqlcipher.py | 705 - 13 files changed, 15557 insertions(+), 15523 deletions(-) delete mode 100644 testing/tests/client/hacker_crackdown.txt create mode 100644 testing/tests/couch/common.py create mode 100644 testing/tests/couch/test_atomicity.py create mode 100644 testing/tests/couch/test_backend.py create mode 100644 testing/tests/couch/test_command.py delete mode 100644 testing/tests/couch/test_couch.py delete mode 100644 testing/tests/couch/test_couch_operations_atomicity.py create mode 100644 testing/tests/couch/test_ddocs.py create mode 100644 testing/tests/couch/test_sync.py create mode 100644 testing/tests/couch/test_sync_target.py create mode 100644 testing/tests/sqlcipher/hacker_crackdown.txt create mode 100644 testing/tests/sqlcipher/test_backend.py delete mode 100644 testing/tests/sqlcipher/test_sqlcipher.py diff --git a/testing/tests/client/hacker_crackdown.txt b/testing/tests/client/hacker_crackdown.txt deleted file mode 100644 index a01eb509..00000000 --- a/testing/tests/client/hacker_crackdown.txt +++ /dev/null @@ -1,13005 +0,0 @@ -The Project Gutenberg EBook of Hacker Crackdown, by Bruce Sterling - -This eBook is for the use of anyone anywhere at no cost and with -almost no restrictions whatsoever. You may copy it, give it away or -re-use it under the terms of the Project Gutenberg License included -with this eBook or online at www.gutenberg.org - -** This is a COPYRIGHTED Project Gutenberg eBook, Details Below ** -** Please follow the copyright guidelines in this file. ** - -Title: Hacker Crackdown - Law and Disorder on the Electronic Frontier - -Author: Bruce Sterling - -Posting Date: February 9, 2012 [EBook #101] -Release Date: January, 1994 - -Language: English - - -*** START OF THIS PROJECT GUTENBERG EBOOK HACKER CRACKDOWN *** - - - - - - - - - - - - - -THE HACKER CRACKDOWN - -Law and Disorder on the Electronic Frontier - -by Bruce Sterling - - - - -CONTENTS - - -Preface to the Electronic Release of The Hacker Crackdown - -Chronology of the Hacker Crackdown - - -Introduction - - -Part 1: CRASHING THE SYSTEM -A Brief History of Telephony -Bell's Golden Vaporware -Universal Service -Wild Boys and Wire Women -The Electronic Communities -The Ungentle Giant -The Breakup -In Defense of the System -The Crash Post-Mortem -Landslides in Cyberspace - - -Part 2: THE DIGITAL UNDERGROUND -Steal This Phone -Phreaking and Hacking -The View From Under the Floorboards -Boards: Core of the Underground -Phile Phun -The Rake's Progress -Strongholds of the Elite -Sting Boards -Hot Potatoes -War on the Legion -Terminus -Phile 9-1-1 -War Games -Real Cyberpunk - - -Part 3: LAW AND ORDER -Crooked Boards -The World's Biggest Hacker Bust -Teach Them a Lesson -The U.S. Secret Service -The Secret Service Battles the Boodlers -A Walk Downtown -FCIC: The Cutting-Edge Mess -Cyberspace Rangers -FLETC: Training the Hacker-Trackers - - -Part 4: THE CIVIL LIBERTARIANS -NuPrometheus + FBI = Grateful Dead -Whole Earth + Computer Revolution = WELL -Phiber Runs Underground and Acid Spikes the Well -The Trial of Knight Lightning -Shadowhawk Plummets to Earth -Kyrie in the Confessional -$79,499 -A Scholar Investigates -Computers, Freedom, and Privacy - - -Electronic Afterword to The Hacker Crackdown, Halloween 1993 - - - - -THE HACKER CRACKDOWN - -Law and Disorder on the Electronic Frontier - -by Bruce Sterling - - - - - -Preface to the Electronic Release of The Hacker Crackdown - - -January 1, 1994--Austin, Texas - - -Hi, I'm Bruce Sterling, the author of this electronic book. - -Out in the traditional world of print, The Hacker Crackdown -is ISBN 0-553-08058-X, and is formally catalogued by -the Library of Congress as "1. Computer crimes--United States. -2. Telephone--United States--Corrupt practices. -3. Programming (Electronic computers)--United States--Corrupt practices." - -`Corrupt practices,' I always get a kick out of that description. -Librarians are very ingenious people. - -The paperback is ISBN 0-553-56370-X. If you go -and buy a print version of The Hacker Crackdown, -an action I encourage heartily, you may notice that -in the front of the book, beneath the copyright notice-- -"Copyright (C) 1992 by Bruce Sterling"-- -it has this little block of printed legal -boilerplate from the publisher. It says, and I quote: - - "No part of this book may be reproduced or transmitted in any form -or by any means, electronic or mechanical, including photocopying, -recording, or by any information storage and retrieval system, -without permission in writing from the publisher. -For information address: Bantam Books." - -This is a pretty good disclaimer, as such disclaimers go. -I collect intellectual-property disclaimers, and I've seen dozens of them, -and this one is at least pretty straightforward. In this narrow -and particular case, however, it isn't quite accurate. -Bantam Books puts that disclaimer on every book they publish, -but Bantam Books does not, in fact, own the electronic rights to this book. -I do, because of certain extensive contract maneuverings my agent and I -went through before this book was written. I want to give those electronic -publishing rights away through certain not-for-profit channels, -and I've convinced Bantam that this is a good idea. - -Since Bantam has seen fit to peacably agree to this scheme of mine, -Bantam Books is not going to fuss about this. Provided you don't try -to sell the book, they are not going to bother you for what you do with -the electronic copy of this book. If you want to check this out personally, -you can ask them; they're at 1540 Broadway NY NY 10036. However, if you were -so foolish as to print this book and start retailing it for money in violation -of my copyright and the commercial interests of Bantam Books, then Bantam, -a part of the gigantic Bertelsmann multinational publishing combine, -would roust some of their heavy-duty attorneys out of hibernation -and crush you like a bug. This is only to be expected. -I didn't write this book so that you could make money out of it. -If anybody is gonna make money out of this book, -it's gonna be me and my publisher. - -My publisher deserves to make money out of this book. -Not only did the folks at Bantam Books commission me -to write the book, and pay me a hefty sum to do so, but -they bravely printed, in text, an electronic document the -reproduction of which was once alleged to be a federal felony. -Bantam Books and their numerous attorneys were very brave -and forthright about this book. Furthermore, my former editor -at Bantam Books, Betsy Mitchell, genuinely cared about this project, -and worked hard on it, and had a lot of wise things to say -about the manuscript. Betsy deserves genuine credit for this book, -credit that editors too rarely get. - -The critics were very kind to The Hacker Crackdown, -and commercially the book has done well. On the other hand, -I didn't write this book in order to squeeze every last nickel -and dime out of the mitts of impoverished sixteen-year-old -cyberpunk high-school-students. Teenagers don't have any money-- -(no, not even enough for the six-dollar Hacker Crackdown paperback, -with its attractive bright-red cover and useful index). -That's a major reason why teenagers sometimes succumb to the temptation -to do things they shouldn't, such as swiping my books out of libraries. -Kids: this one is all yours, all right? Go give the print version back. -*8-) - -Well-meaning, public-spirited civil libertarians don't have much money, -either. And it seems almost criminal to snatch cash out of the hands of -America's direly underpaid electronic law enforcement community. - -If you're a computer cop, a hacker, or an electronic civil -liberties activist, you are the target audience for this book. -I wrote this book because I wanted to help you, and help other people -understand you and your unique, uhm, problems. I wrote this book -to aid your activities, and to contribute to the public discussion -of important political issues. In giving the text away in this -fashion, I am directly contributing to the book's ultimate aim: -to help civilize cyberspace. - -Information WANTS to be free. And the information inside -this book longs for freedom with a peculiar intensity. -I genuinely believe that the natural habitat of this book -is inside an electronic network. That may not be the easiest -direct method to generate revenue for the book's author, -but that doesn't matter; this is where this book belongs -by its nature. I've written other books--plenty of other books-- -and I'll write more and I am writing more, but this one is special. -I am making The Hacker Crackdown available electronically -as widely as I can conveniently manage, and if you like the book, -and think it is useful, then I urge you to do the same with it. - -You can copy this electronic book. Copy the heck out of it, -be my guest, and give those copies to anybody who wants them. -The nascent world of cyberspace is full of sysadmins, teachers, -trainers, cybrarians, netgurus, and various species of cybernetic activist. -If you're one of those people, I know about you, and I know the hassle -you go through to try to help people learn about the electronic frontier. -I hope that possessing this book in electronic form will lessen your troubles. -Granted, this treatment of our electronic social spectrum is not the ultimate -in academic rigor. And politically, it has something to offend -and trouble almost everyone. But hey, I'm told it's readable, -and at least the price is right. - -You can upload the book onto bulletin board systems, or Internet nodes, -or electronic discussion groups. Go right ahead and do that, I am giving -you express permission right now. Enjoy yourself. - -You can put the book on disks and give the disks away, -as long as you don't take any money for it. - -But this book is not public domain. You can't copyright it in -your own name. I own the copyright. Attempts to pirate this book -and make money from selling it may involve you in a serious litigative snarl. -Believe me, for the pittance you might wring out of such an action, -it's really not worth it. This book don't "belong" to you. -In an odd but very genuine way, I feel it doesn't "belong" to me, either. -It's a book about the people of cyberspace, and distributing it in this way -is the best way I know to actually make this information available, -freely and easily, to all the people of cyberspace--including people -far outside the borders of the United States, who otherwise may never -have a chance to see any edition of the book, and who may perhaps learn -something useful from this strange story of distant, obscure, but portentous -events in so-called "American cyberspace." - -This electronic book is now literary freeware. It now belongs to the -emergent realm of alternative information economics. You have no right -to make this electronic book part of the conventional flow of commerce. -Let it be part of the flow of knowledge: there's a difference. -I've divided the book into four sections, so that it is less ungainly -for upload and download; if there's a section of particular relevance -to you and your colleagues, feel free to reproduce that one and skip the rest. - -[Project Gutenberg has reassembled the file, with Sterling's permission.] - -Just make more when you need them, and give them to whoever might want them. - -Now have fun. - -Bruce Sterling--bruces@well.sf.ca.us - - -THE HACKER CRACKDOWN - -Law and Disorder on the Electronic Frontier - -by Bruce Sterling - - - - - - - -CHRONOLOGY OF THE HACKER CRACKDOWN - - -1865 U.S. Secret Service (USSS) founded. - -1876 Alexander Graham Bell invents telephone. - -1878 First teenage males flung off phone system by enraged authorities. - -1939 "Futurian" science-fiction group raided by Secret Service. - -1971 Yippie phone phreaks start YIPL/TAP magazine. - -1972 RAMPARTS magazine seized in blue-box rip-off scandal. - -1978 Ward Christenson and Randy Suess create first personal - computer bulletin board system. - -1982 William Gibson coins term "cyberspace." - -1982 "414 Gang" raided. - -1983-1983 AT&T dismantled in divestiture. - -1984 Congress passes Comprehensive Crime Control Act giving USSS - jurisdiction over credit card fraud and computer fraud. - -1984 "Legion of Doom" formed. - -1984. 2600: THE HACKER QUARTERLY founded. - -1984. WHOLE EARTH SOFTWARE CATALOG published. - -1985. First police "sting" bulletin board systems established. - -1985. Whole Earth 'Lectronic Link computer conference (WELL) goes on-line. - -1986 Computer Fraud and Abuse Act passed. - -1986 Electronic Communications Privacy Act passed. - -1987 Chicago prosecutors form Computer Fraud and Abuse Task Force. - - -1988 - -July. Secret Service covertly videotapes "SummerCon" hacker convention. - -September. "Prophet" cracks BellSouth AIMSX computer network - and downloads E911 Document to his own computer and to Jolnet. - -September. AT&T Corporate Information Security informed of Prophet's action. - -October. Bellcore Security informed of Prophet's action. - - -1989 - -January. Prophet uploads E911 Document to Knight Lightning. - -February 25. Knight Lightning publishes E911 Document in PHRACK - electronic newsletter. - -May. Chicago Task Force raids and arrests "Kyrie." - -June. "NuPrometheus League" distributes Apple Computer proprietary software. - -June 13. Florida probation office crossed with phone-sex line - in switching-station stunt. - -July. "Fry Guy" raided by USSS and Chicago Computer Fraud - and Abuse Task Force. - -July. Secret Service raids "Prophet," "Leftist," and "Urvile" in Georgia. - - -1990 - -January 15. Martin Luther King Day Crash strikes AT&T long-distance - network nationwide. - -January 18-19. Chicago Task Force raids Knight Lightning in St. Louis. - -January 24. USSS and New York State Police raid "Phiber Optik," - "Acid Phreak," and "Scorpion" in New York City. - -February 1. USSS raids "Terminus" in Maryland. - -February 3. Chicago Task Force raids Richard Andrews' home. - -February 6. Chicago Task Force raids Richard Andrews' business. - -February 6. USSS arrests Terminus, Prophet, Leftist, and Urvile. - -February 9. Chicago Task Force arrests Knight Lightning. - -February 20. AT&T Security shuts down public-access - "attctc" computer in Dallas. - -February 21. Chicago Task Force raids Robert Izenberg in Austin. - -March 1. Chicago Task Force raids Steve Jackson Games, Inc., - "Mentor," and "Erik Bloodaxe" in Austin. - -May 7,8,9. - -USSS and Arizona Organized Crime and Racketeering Bureau conduct -"Operation Sundevil" raids in Cincinnatti, Detroit, Los Angeles, -Miami, Newark, Phoenix, Pittsburgh, Richmond, Tucson, San Diego, -San Jose, and San Francisco. - -May. FBI interviews John Perry Barlow re NuPrometheus case. - -June. Mitch Kapor and Barlow found Electronic Frontier Foundation; - Barlow publishes CRIME AND PUZZLEMENT manifesto. - -July 24-27. Trial of Knight Lightning. - -1991 - -February. CPSR Roundtable in Washington, D.C. - -March 25-28. Computers, Freedom and Privacy conference in San Francisco. - -May 1. Electronic Frontier Foundation, Steve Jackson, - and others file suit against members of Chicago Task Force. - -July 1-2. Switching station phone software crash affects - Washington, Los Angeles, Pittsburgh, San Francisco. - -September 17. AT&T phone crash affects New York City and three airports. - - - - -Introduction - -This is a book about cops, and wild teenage whiz-kids, and lawyers, -and hairy-eyed anarchists, and industrial technicians, and hippies, -and high-tech millionaires, and game hobbyists, and computer security -experts, and Secret Service agents, and grifters, and thieves. - -This book is about the electronic frontier of the 1990s. -It concerns activities that take place inside computers -and over telephone lines. - -A science fiction writer coined the useful term "cyberspace" in 1982, -but the territory in question, the electronic frontier, is about -a hundred and thirty years old. Cyberspace is the "place" where -a telephone conversation appears to occur. Not inside your actual phone, -the plastic device on your desk. Not inside the other person's phone, -in some other city. THE PLACE BETWEEN the phones. The indefinite -place OUT THERE, where the two of you, two human beings, -actually meet and communicate. - -Although it is not exactly "real," "cyberspace" is a genuine place. -Things happen there that have very genuine consequences. This "place" -is not "real," but it is serious, it is earnest. Tens of thousands -of people have dedicated their lives to it, to the public service -of public communication by wire and electronics. - -People have worked on this "frontier" for generations now. -Some people became rich and famous from their efforts there. -Some just played in it, as hobbyists. Others soberly pondered it, -and wrote about it, and regulated it, and negotiated over it in -international forums, and sued one another about it, in gigantic, -epic court battles that lasted for years. And almost since -the beginning, some people have committed crimes in this place. - -But in the past twenty years, this electrical "space," -which was once thin and dark and one-dimensional--little more -than a narrow speaking-tube, stretching from phone to phone-- -has flung itself open like a gigantic jack-in-the-box. -Light has flooded upon it, the eerie light of the glowing computer screen. -This dark electric netherworld has become a vast flowering electronic landscape. -Since the 1960s, the world of the telephone has cross-bred itself -with computers and television, and though there is still no substance -to cyberspace, nothing you can handle, it has a strange kind -of physicality now. It makes good sense today to talk of cyberspace -as a place all its own. - -Because people live in it now. Not just a few people, -not just a few technicians and eccentrics, but thousands -of people, quite normal people. And not just for a little while, -either, but for hours straight, over weeks, and months, -and years. Cyberspace today is a "Net," a "Matrix," -international in scope and growing swiftly and steadily. -It's growing in size, and wealth, and political importance. - -People are making entire careers in modern cyberspace. -Scientists and technicians, of course; they've been there -for twenty years now. But increasingly, cyberspace -is filling with journalists and doctors and lawyers -and artists and clerks. Civil servants make their -careers there now, "on-line" in vast government data-banks; -and so do spies, industrial, political, and just plain snoops; -and so do police, at least a few of them. And there are children -living there now. - -People have met there and been married there. -There are entire living communities in cyberspace today; -chattering, gossiping, planning, conferring and scheming, -leaving one another voice-mail and electronic mail, -giving one another big weightless chunks of valuable data, -both legitimate and illegitimate. They busily pass one another -computer software and the occasional festering computer virus. - -We do not really understand how to live in cyberspace yet. -We are feeling our way into it, blundering about. -That is not surprising. Our lives in the physical world, -the "real" world, are also far from perfect, despite a lot more practice. -Human lives, real lives, are imperfect by their nature, and there are -human beings in cyberspace. The way we live in cyberspace is -a funhouse mirror of the way we live in the real world. -We take both our advantages and our troubles with us. - -This book is about trouble in cyberspace. -Specifically, this book is about certain strange events in -the year 1990, an unprecedented and startling year for the -the growing world of computerized communications. - -In 1990 there came a nationwide crackdown on illicit -computer hackers, with arrests, criminal charges, -one dramatic show-trial, several guilty pleas, and -huge confiscations of data and equipment all over the USA. - -The Hacker Crackdown of 1990 was larger, better organized, -more deliberate, and more resolute than any previous effort -in the brave new world of computer crime. The U.S. Secret Service, -private telephone security, and state and local law enforcement groups -across the country all joined forces in a determined attempt to break -the back of America's electronic underground. It was a fascinating -effort, with very mixed results. - -The Hacker Crackdown had another unprecedented effect; -it spurred the creation, within "the computer community," -of the Electronic Frontier Foundation, a new and very odd -interest group, fiercely dedicated to the establishment -and preservation of electronic civil liberties. The crackdown, -remarkable in itself, has created a melee of debate over electronic crime, -punishment, freedom of the press, and issues of search and seizure. -Politics has entered cyberspace. Where people go, politics follow. - -This is the story of the people of cyberspace. - - - -PART ONE: Crashing the System - -On January 15, 1990, AT&T's long-distance telephone switching system crashed. - -This was a strange, dire, huge event. Sixty thousand people lost -their telephone service completely. During the nine long hours -of frantic effort that it took to restore service, some seventy million -telephone calls went uncompleted. - -Losses of service, known as "outages" in the telco trade, -are a known and accepted hazard of the telephone business. -Hurricanes hit, and phone cables get snapped by the thousands. -Earthquakes wrench through buried fiber-optic lines. -Switching stations catch fire and burn to the ground. -These things do happen. There are contingency plans for them, -and decades of experience in dealing with them. -But the Crash of January 15 was unprecedented. -It was unbelievably huge, and it occurred for -no apparent physical reason. - -The crash started on a Monday afternoon in a single -switching-station in Manhattan. But, unlike any merely -physical damage, it spread and spread. Station after -station across America collapsed in a chain reaction, -until fully half of AT&T's network had gone haywire -and the remaining half was hard-put to handle the overflow. - -Within nine hours, AT&T software engineers more or less -understood what had caused the crash. Replicating the -problem exactly, poring over software line by line, -took them a couple of weeks. But because it was hard -to understand technically, the full truth of the matter -and its implications were not widely and thoroughly aired -and explained. The root cause of the crash remained obscure, -surrounded by rumor and fear. - -The crash was a grave corporate embarrassment. -The "culprit" was a bug in AT&T's own software--not the -sort of admission the telecommunications giant wanted -to make, especially in the face of increasing competition. -Still, the truth WAS told, in the baffling technical terms -necessary to explain it. - -Somehow the explanation failed to persuade -American law enforcement officials and even telephone -corporate security personnel. These people were not -technical experts or software wizards, and they had their -own suspicions about the cause of this disaster. - -The police and telco security had important sources -of information denied to mere software engineers. -They had informants in the computer underground and -years of experience in dealing with high-tech rascality -that seemed to grow ever more sophisticated. -For years they had been expecting a direct and -savage attack against the American national telephone system. -And with the Crash of January 15--the first month of a -new, high-tech decade--their predictions, fears, -and suspicions seemed at last to have entered the real world. -A world where the telephone system had not merely crashed, -but, quite likely, BEEN crashed--by "hackers." - -The crash created a large dark cloud of suspicion -that would color certain people's assumptions and actions -for months. The fact that it took place in the realm of -software was suspicious on its face. The fact that it -occurred on Martin Luther King Day, still the most -politically touchy of American holidays, made it more -suspicious yet. - -The Crash of January 15 gave the Hacker Crackdown -its sense of edge and its sweaty urgency. It made people, -powerful people in positions of public authority, -willing to believe the worst. And, most fatally, -it helped to give investigators a willingness -to take extreme measures and the determination -to preserve almost total secrecy. - -An obscure software fault in an aging switching system -in New York was to lead to a chain reaction of legal -and constitutional trouble all across the country. - -# - -Like the crash in the telephone system, this chain reaction -was ready and waiting to happen. During the 1980s, -the American legal system was extensively patched -to deal with the novel issues of computer crime. -There was, for instance, the Electronic Communications -Privacy Act of 1986 (eloquently described as "a stinking mess" -by a prominent law enforcement official). And there was the -draconian Computer Fraud and Abuse Act of 1986, passed unanimously -by the United States Senate, which later would reveal -a large number of flaws. Extensive, well-meant efforts -had been made to keep the legal system up to date. -But in the day-to-day grind of the real world, -even the most elegant software tends to crumble -and suddenly reveal its hidden bugs. - -Like the advancing telephone system, the American legal system -was certainly not ruined by its temporary crash; but for those -caught under the weight of the collapsing system, life became -a series of blackouts and anomalies. - -In order to understand why these weird events occurred, -both in the world of technology and in the world of law, -it's not enough to understand the merely technical problems. -We will get to those; but first and foremost, we must try -to understand the telephone, and the business of telephones, -and the community of human beings that telephones have created. - -# - -Technologies have life cycles, like cities do, -like institutions do, like laws and governments do. - -The first stage of any technology is the Question -Mark, often known as the "Golden Vaporware" stage. -At this early point, the technology is only a phantom, -a mere gleam in the inventor's eye. One such inventor -was a speech teacher and electrical tinkerer named -Alexander Graham Bell. - -Bell's early inventions, while ingenious, failed to move the world. -In 1863, the teenage Bell and his brother Melville made an artificial -talking mechanism out of wood, rubber, gutta-percha, and tin. -This weird device had a rubber-covered "tongue" made of movable -wooden segments, with vibrating rubber "vocal cords," and -rubber "lips" and "cheeks." While Melville puffed a bellows -into a tin tube, imitating the lungs, young Alec Bell would -manipulate the "lips," "teeth," and "tongue," causing the thing -to emit high-pitched falsetto gibberish. - -Another would-be technical breakthrough was the Bell "phonautograph" -of 1874, actually made out of a human cadaver's ear. Clamped into place -on a tripod, this grisly gadget drew sound-wave images on smoked glass -through a thin straw glued to its vibrating earbones. - -By 1875, Bell had learned to produce audible sounds--ugly shrieks -and squawks--by using magnets, diaphragms, and electrical current. - -Most "Golden Vaporware" technologies go nowhere. - -But the second stage of technology is the Rising Star, -or, the "Goofy Prototype," stage. The telephone, Bell's -most ambitious gadget yet, reached this stage on March -10, 1876. On that great day, Alexander Graham Bell -became the first person to transmit intelligible human -speech electrically. As it happened, young Professor Bell, -industriously tinkering in his Boston lab, had spattered -his trousers with acid. His assistant, Mr. Watson, -heard his cry for help--over Bell's experimental -audio-telegraph. This was an event without precedent. - -Technologies in their "Goofy Prototype" stage rarely -work very well. They're experimental, and therefore -half- baked and rather frazzled. The prototype may -be attractive and novel, and it does look as if it ought -to be good for something-or-other. But nobody, including -the inventor, is quite sure what. Inventors, and speculators, -and pundits may have very firm ideas about its potential -use, but those ideas are often very wrong. - -The natural habitat of the Goofy Prototype is in trade shows -and in the popular press. Infant technologies need publicity -and investment money like a tottering calf need milk. -This was very true of Bell's machine. To raise research and -development money, Bell toured with his device as a stage attraction. - -Contemporary press reports of the stage debut of the telephone -showed pleased astonishment mixed with considerable dread. -Bell's stage telephone was a large wooden box with a crude -speaker-nozzle, the whole contraption about the size and shape -of an overgrown Brownie camera. Its buzzing steel soundplate, -pumped up by powerful electromagnets, was loud enough to fill -an auditorium. Bell's assistant Mr. Watson, who could manage -on the keyboards fairly well, kicked in by playing the organ -from distant rooms, and, later, distant cities. This feat was -considered marvellous, but very eerie indeed. - -Bell's original notion for the telephone, an idea promoted -for a couple of years, was that it would become a mass medium. -We might recognize Bell's idea today as something close to modern -"cable radio." Telephones at a central source would transmit music, -Sunday sermons, and important public speeches to a paying network -of wired-up subscribers. - -At the time, most people thought this notion made good sense. -In fact, Bell's idea was workable. In Hungary, this philosophy -of the telephone was successfully put into everyday practice. -In Budapest, for decades, from 1893 until after World War I, -there was a government-run information service called -"Telefon Hirmondo-." Hirmondo- was a centralized source -of news and entertainment and culture, including stock reports, -plays, concerts, and novels read aloud. At certain hours -of the day, the phone would ring, you would plug in -a loudspeaker for the use of the family, and Telefon -Hirmondo- would be on the air--or rather, on the phone. - -Hirmondo- is dead tech today, but Hirmondo- might be considered -a spiritual ancestor of the modern telephone-accessed computer -data services, such as CompuServe, GEnie or Prodigy. -The principle behind Hirmondo- is also not too far from computer -"bulletin- board systems" or BBS's, which arrived in the late 1970s, -spread rapidly across America, and will figure largely in this book. - -We are used to using telephones for individual person-to-person speech, -because we are used to the Bell system. But this was just one possibility -among many. Communication networks are very flexible and protean, -especially when their hardware becomes sufficiently advanced. -They can be put to all kinds of uses. And they have been-- -and they will be. - -Bell's telephone was bound for glory, but this was a combination -of political decisions, canny infighting in court, inspired industrial -leadership, receptive local conditions and outright good luck. -Much the same is true of communications systems today. - -As Bell and his backers struggled to install their newfangled system -in the real world of nineteenth-century New England, they had to fight -against skepticism and industrial rivalry. There was already a strong -electrical communications network present in America: the telegraph. -The head of the Western Union telegraph system dismissed Bell's prototype -as "an electrical toy" and refused to buy the rights to Bell's patent. -The telephone, it seemed, might be all right as a parlor entertainment-- -but not for serious business. - -Telegrams, unlike mere telephones, left a permanent physical record -of their messages. Telegrams, unlike telephones, could be answered -whenever the recipient had time and convenience. And the telegram -had a much longer distance-range than Bell's early telephone. -These factors made telegraphy seem a much more sound and businesslike -technology--at least to some. - -The telegraph system was huge, and well-entrenched. -In 1876, the United States had 214,000 miles of telegraph wire, -and 8500 telegraph offices. There were specialized telegraphs -for businesses and stock traders, government, police and fire departments. -And Bell's "toy" was best known as a stage-magic musical device. - -The third stage of technology is known as the "Cash Cow" stage. -In the "cash cow" stage, a technology finds its place in the world, -and matures, and becomes settled and productive. After a year or so, -Alexander Graham Bell and his capitalist backers concluded that -eerie music piped from nineteenth-century cyberspace was not the real -selling-point of his invention. Instead, the telephone was about speech-- -individual, personal speech, the human voice, human conversation and -human interaction. The telephone was not to be managed from any centralized -broadcast center. It was to be a personal, intimate technology. - -When you picked up a telephone, you were not absorbing -the cold output of a machine--you were speaking to another human being. -Once people realized this, their instinctive dread of the telephone -as an eerie, unnatural device, swiftly vanished. A "telephone call" -was not a "call" from a "telephone" itself, but a call from another -human being, someone you would generally know and recognize. -The real point was not what the machine could do for you (or to you), -but what you yourself, a person and citizen, could do THROUGH the machine. -This decision on the part of the young Bell Company was absolutely vital. - -The first telephone networks went up around Boston--mostly among -the technically curious and the well-to-do (much the same segment -of the American populace that, a hundred years later, would be -buying personal computers). Entrenched backers of the telegraph -continued to scoff. - -But in January 1878, a disaster made the telephone famous. -A train crashed in Tarriffville, Connecticut. Forward-looking -doctors in the nearby city of Hartford had had Bell's -"speaking telephone" installed. An alert local druggist -was able to telephone an entire community of local doctors, -who rushed to the site to give aid. The disaster, as disasters do, -aroused intense press coverage. The phone had proven its usefulness -in the real world. - -After Tarriffville, the telephone network spread like crabgrass. -By 1890 it was all over New England. By '93, out to Chicago. -By '97, into Minnesota, Nebraska and Texas. By 1904 it was -all over the continent. - -The telephone had become a mature technology. Professor Bell -(now generally known as "Dr. Bell" despite his lack of a formal degree) -became quite wealthy. He lost interest in the tedious day-to-day business -muddle of the booming telephone network, and gratefully returned -his attention to creatively hacking-around in his various laboratories, -which were now much larger, better-ventilated, and gratifyingly -better-equipped. Bell was never to have another great inventive success, -though his speculations and prototypes anticipated fiber-optic transmission, -manned flight, sonar, hydrofoil ships, tetrahedral construction, and -Montessori education. The "decibel," the standard scientific measure -of sound intensity, was named after Bell. - -Not all Bell's vaporware notions were inspired. He was fascinated -by human eugenics. He also spent many years developing a weird personal -system of astrophysics in which gravity did not exist. - -Bell was a definite eccentric. He was something of a hypochondriac, -and throughout his life he habitually stayed up until four A.M., -refusing to rise before noon. But Bell had accomplished a great feat; -he was an idol of millions and his influence, wealth, and great -personal charm, combined with his eccentricity, made him something -of a loose cannon on deck. Bell maintained a thriving scientific -salon in his winter mansion in Washington, D.C., which gave him -considerable backstage influence in governmental and scientific circles. -He was a major financial backer of the the magazines Science and -National Geographic, both still flourishing today as important organs -of the American scientific establishment. - -Bell's companion Thomas Watson, similarly wealthy and similarly odd, -became the ardent political disciple of a 19th-century science-fiction writer -and would-be social reformer, Edward Bellamy. Watson also trod the boards -briefly as a Shakespearian actor. - -There would never be another Alexander Graham Bell, -but in years to come there would be surprising numbers -of people like him. Bell was a prototype of the -high-tech entrepreneur. High-tech entrepreneurs will -play a very prominent role in this book: not merely as -technicians and businessmen, but as pioneers of the -technical frontier, who can carry the power and prestige -they derive from high-technology into the political and -social arena. - -Like later entrepreneurs, Bell was fierce in defense of -his own technological territory. As the telephone began to -flourish, Bell was soon involved in violent lawsuits in the -defense of his patents. Bell's Boston lawyers were -excellent, however, and Bell himself, as an elocution -teacher and gifted public speaker, was a devastatingly -effective legal witness. In the eighteen years of Bell's patents, -the Bell company was involved in six hundred separate lawsuits. -The legal records printed filled 149 volumes. The Bell Company -won every single suit. - -After Bell's exclusive patents expired, rival telephone -companies sprang up all over America. Bell's company, -American Bell Telephone, was soon in deep trouble. -In 1907, American Bell Telephone fell into the hands of the -rather sinister J.P. Morgan financial cartel, robber-baron -speculators who dominated Wall Street. - -At this point, history might have taken a different turn. -American might well have been served forever by a patchwork -of locally owned telephone companies. Many state politicians -and local businessmen considered this an excellent solution. - -But the new Bell holding company, American Telephone and Telegraph -or AT&T, put in a new man at the helm, a visionary industrialist -named Theodore Vail. Vail, a former Post Office manager, -understood large organizations and had an innate feeling -for the nature of large-scale communications. Vail quickly -saw to it that AT&T seized the technological edge once again. -The Pupin and Campbell "loading coil," and the deForest -"audion," are both extinct technology today, but in 1913 -they gave Vail's company the best LONG-DISTANCE lines -ever built. By controlling long-distance--the links -between, and over, and above the smaller local phone -companies--AT&T swiftly gained the whip-hand over them, -and was soon devouring them right and left. - -Vail plowed the profits back into research and development, -starting the Bell tradition of huge-scale and brilliant -industrial research. - -Technically and financially, AT&T gradually steamrollered -the opposition. Independent telephone companies never -became entirely extinct, and hundreds of them flourish today. -But Vail's AT&T became the supreme communications company. -At one point, Vail's AT&T bought Western Union itself, -the very company that had derided Bell's telephone as a "toy." -Vail thoroughly reformed Western Union's hidebound business -along his modern principles; but when the federal government -grew anxious at this centralization of power, Vail politely -gave Western Union back. - -This centralizing process was not unique. Very similar -events had happened in American steel, oil, and railroads. -But AT&T, unlike the other companies, was to remain supreme. -The monopoly robber-barons of those other industries -were humbled and shattered by government trust-busting. - -Vail, the former Post Office official, was quite willing -to accommodate the US government; in fact he would -forge an active alliance with it. AT&T would become -almost a wing of the American government, almost -another Post Office--though not quite. AT&T would -willingly submit to federal regulation, but in return, -it would use the government's regulators as its own police, -who would keep out competitors and assure the Bell -system's profits and preeminence. - -This was the second birth--the political birth--of the -American telephone system. Vail's arrangement was to -persist, with vast success, for many decades, until 1982. -His system was an odd kind of American industrial socialism. -It was born at about the same time as Leninist Communism, -and it lasted almost as long--and, it must be admitted, -to considerably better effect. - -Vail's system worked. Except perhaps for aerospace, -there has been no technology more thoroughly dominated -by Americans than the telephone. The telephone was -seen from the beginning as a quintessentially American -technology. Bell's policy, and the policy of Theodore Vail, -was a profoundly democratic policy of UNIVERSAL ACCESS. -Vail's famous corporate slogan, "One Policy, One System, -Universal Service," was a political slogan, with a very -American ring to it. - -The American telephone was not to become the specialized tool -of government or business, but a general public utility. -At first, it was true, only the wealthy could afford -private telephones, and Bell's company pursued the -business markets primarily. The American phone system -was a capitalist effort, meant to make money; it was not a charity. -But from the first, almost all communities with telephone service -had public telephones. And many stores--especially drugstores-- -offered public use of their phones. You might not own a telephone-- -but you could always get into the system, if you really needed to. - -There was nothing inevitable about this decision to make telephones -"public" and "universal." Vail's system involved a profound act -of trust in the public. This decision was a political one, -informed by the basic values of the American republic. -The situation might have been very different; -and in other countries, under other systems, -it certainly was. - -Joseph Stalin, for instance, vetoed plans for a Soviet -phone system soon after the Bolshevik revolution. -Stalin was certain that publicly accessible telephones -would become instruments of anti-Soviet counterrevolution -and conspiracy. (He was probably right.) When telephones -did arrive in the Soviet Union, they would be instruments -of Party authority, and always heavily tapped. (Alexander -Solzhenitsyn's prison-camp novel The First Circle -describes efforts to develop a phone system more suited -to Stalinist purposes.) - -France, with its tradition of rational centralized government, -had fought bitterly even against the electric telegraph, -which seemed to the French entirely too anarchical and frivolous. -For decades, nineteenth-century France communicated via the -"visual telegraph," a nation-spanning, government-owned semaphore -system of huge stone towers that signalled from hilltops, -across vast distances, with big windmill-like arms. -In 1846, one Dr. Barbay, a semaphore enthusiast, -memorably uttered an early version of what might be called -"the security expert's argument" against the open media. - -"No, the electric telegraph is not a sound invention. -It will always be at the mercy of the slightest disruption, -wild youths, drunkards, bums, etc. . . . The electric telegraph -meets those destructive elements with only a few meters of wire -over which supervision is impossible. A single man could, -without being seen, cut the telegraph wires leading to Paris, -and in twenty-four hours cut in ten different places the wires -of the same line, without being arrested. The visual telegraph, -on the contrary, has its towers, its high walls, its gates -well-guarded from inside by strong armed men. Yes, I declare, -substitution of the electric telegraph for the visual one -is a dreadful measure, a truly idiotic act." - -Dr. Barbay and his high-security stone machines -were eventually unsuccessful, but his argument-- -that communication exists for the safety and convenience -of the state, and must be carefully protected from the wild -boys and the gutter rabble who might want to crash the -system--would be heard again and again. - -When the French telephone system finally did arrive, -its snarled inadequacy was to be notorious. Devotees -of the American Bell System often recommended a trip -to France, for skeptics. - -In Edwardian Britain, issues of class and privacy -were a ball-and-chain for telephonic progress. It was -considered outrageous that anyone--any wild fool off -the street--could simply barge bellowing into one's office -or home, preceded only by the ringing of a telephone bell. -In Britain, phones were tolerated for the use of business, -but private phones tended be stuffed away into closets, -smoking rooms, or servants' quarters. Telephone operators -were resented in Britain because they did not seem to -"know their place." And no one of breeding would print -a telephone number on a business card; this seemed a crass -attempt to make the acquaintance of strangers. - -But phone access in America was to become a popular right; -something like universal suffrage, only more so. -American women could not yet vote when the phone system -came through; yet from the beginning American women -doted on the telephone. This "feminization" of the -American telephone was often commented on by foreigners. -Phones in America were not censored or stiff or formalized; -they were social, private, intimate, and domestic. -In America, Mother's Day is by far the busiest day -of the year for the phone network. - -The early telephone companies, and especially AT&T, -were among the foremost employers of American women. -They employed the daughters of the American middle-class -in great armies: in 1891, eight thousand women; by 1946, -almost a quarter of a million. Women seemed to enjoy -telephone work; it was respectable, it was steady, -it paid fairly well as women's work went, and--not least-- -it seemed a genuine contribution to the social good -of the community. Women found Vail's ideal of public -service attractive. This was especially true in rural areas, -where women operators, running extensive rural party-lines, -enjoyed considerable social power. The operator knew everyone -on the party-line, and everyone knew her. - -Although Bell himself was an ardent suffragist, the -telephone company did not employ women for the sake of -advancing female liberation. AT&T did this for sound -commercial reasons. The first telephone operators of -the Bell system were not women, but teenage American boys. -They were telegraphic messenger boys (a group about to -be rendered technically obsolescent), who swept up -around the phone office, dunned customers for bills, -and made phone connections on the switchboard, -all on the cheap. - -Within the very first year of operation, 1878, -Bell's company learned a sharp lesson about combining -teenage boys and telephone switchboards. Putting -teenage boys in charge of the phone system brought swift -and consistent disaster. Bell's chief engineer described them -as "Wild Indians." The boys were openly rude to customers. -They talked back to subscribers, saucing off, -uttering facetious remarks, and generally giving lip. -The rascals took Saint Patrick's Day off without permission. -And worst of all they played clever tricks with -the switchboard plugs: disconnecting calls, crossing lines -so that customers found themselves talking to strangers, -and so forth. - -This combination of power, technical mastery, and effective -anonymity seemed to act like catnip on teenage boys. - -This wild-kid-on-the-wires phenomenon was not confined to -the USA; from the beginning, the same was true of the British -phone system. An early British commentator kindly remarked: -"No doubt boys in their teens found the work not a little irksome, -and it is also highly probable that under the early conditions -of employment the adventurous and inquisitive spirits of which -the average healthy boy of that age is possessed, were not always -conducive to the best attention being given to the wants -of the telephone subscribers." - -So the boys were flung off the system--or at least, -deprived of control of the switchboard. But the -"adventurous and inquisitive spirits" of the teenage boys -would be heard from in the world of telephony, again and again. - -The fourth stage in the technological life-cycle is death: -"the Dog," dead tech. The telephone has so far avoided this fate. -On the contrary, it is thriving, still spreading, still evolving, -and at increasing speed. - -The telephone has achieved a rare and exalted state for a -technological artifact: it has become a HOUSEHOLD OBJECT. -The telephone, like the clock, like pen and paper, -like kitchen utensils and running water, has become -a technology that is visible only by its absence. -The telephone is technologically transparent. -The global telephone system is the largest and most -complex machine in the world, yet it is easy to use. -More remarkable yet, the telephone is almost entirely -physically safe for the user. - -For the average citizen in the 1870s, the telephone -was weirder, more shocking, more "high-tech" and -harder to comprehend, than the most outrageous stunts -of advanced computing for us Americans in the 1990s. -In trying to understand what is happening to us today, -with our bulletin-board systems, direct overseas dialling, -fiber-optic transmissions, computer viruses, hacking stunts, -and a vivid tangle of new laws and new crimes, it is important -to realize that our society has been through a similar challenge before-- -and that, all in all, we did rather well by it. - -Bell's stage telephone seemed bizarre at first. But the -sensations of weirdness vanished quickly, once people began -to hear the familiar voices of relatives and friends, -in their own homes on their own telephones. The telephone -changed from a fearsome high-tech totem to an everyday pillar -of human community. - -This has also happened, and is still happening, -to computer networks. Computer networks such as -NSFnet, BITnet, USENET, JANET, are technically -advanced, intimidating, and much harder to use than -telephones. Even the popular, commercial computer -networks, such as GEnie, Prodigy, and CompuServe, -cause much head-scratching and have been described -as "user-hateful." Nevertheless they too are changing -from fancy high-tech items into everyday sources -of human community. - -The words "community" and "communication" have -the same root. Wherever you put a communications -network, you put a community as well. And whenever -you TAKE AWAY that network--confiscate it, outlaw it, -crash it, raise its price beyond affordability-- -then you hurt that community. - -Communities will fight to defend themselves. People will fight harder -and more bitterly to defend their communities, than they will fight -to defend their own individual selves. And this is very true -of the "electronic community" that arose around computer networks -in the 1980s--or rather, the VARIOUS electronic communities, -in telephony, law enforcement, computing, and the digital -underground that, by the year 1990, were raiding, rallying, -arresting, suing, jailing, fining and issuing angry manifestos. - -None of the events of 1990 were entirely new. -Nothing happened in 1990 that did not have some kind -of earlier and more understandable precedent. What gave -the Hacker Crackdown its new sense of gravity and -importance was the feeling--the COMMUNITY feeling-- -that the political stakes had been raised; that trouble -in cyberspace was no longer mere mischief or inconclusive -skirmishing, but a genuine fight over genuine issues, -a fight for community survival and the shape of the future. - -These electronic communities, having flourished throughout -the 1980s, were becoming aware of themselves, and increasingly, -becoming aware of other, rival communities. Worries were -sprouting up right and left, with complaints, rumors, -uneasy speculations. But it would take a catalyst, a shock, -to make the new world evident. Like Bell's great publicity break, -the Tarriffville Rail Disaster of January 1878, -it would take a cause celebre. - -That cause was the AT&T Crash of January 15, 1990. -After the Crash, the wounded and anxious telephone -community would come out fighting hard. - -# - -The community of telephone technicians, engineers, operators -and researchers is the oldest community in cyberspace. -These are the veterans, the most developed group, -the richest, the most respectable, in most ways the most powerful. -Whole generations have come and gone since Alexander Graham Bell's day, -but the community he founded survives; people work for the phone system -today whose great-grandparents worked for the phone system. -Its specialty magazines, such as Telephony, AT&T Technical Journal, -Telephone Engineer and Management, are decades old; -they make computer publications like Macworld and PC Week -look like amateur johnny-come-latelies. - -And the phone companies take no back seat in high-technology, either. -Other companies' industrial researchers may have won new markets; -but the researchers of Bell Labs have won SEVEN NOBEL PRIZES. -One potent device that Bell Labs originated, the transistor, -has created entire GROUPS of industries. Bell Labs are -world-famous for generating "a patent a day," and have even -made vital discoveries in astronomy, physics and cosmology. - -Throughout its seventy-year history, "Ma Bell" was not so much -a company as a way of life. Until the cataclysmic divestiture -of the 1980s, Ma Bell was perhaps the ultimate maternalist mega-employer. -The AT&T corporate image was the "gentle giant," "the voice with a smile," -a vaguely socialist-realist world of cleanshaven linemen in shiny helmets -and blandly pretty phone-girls in headsets and nylons. Bell System -employees were famous as rock-ribbed Kiwanis and Rotary members, -Little-League enthusiasts, school-board people. - -During the long heyday of Ma Bell, the Bell employee corps -were nurtured top-to-bottom on a corporate ethos of public service. -There was good money in Bell, but Bell was not ABOUT money; -Bell used public relations, but never mere marketeering. -People went into the Bell System for a good life, -and they had a good life. But it was not mere money -that led Bell people out in the midst of storms and earthquakes -to fight with toppled phone-poles, to wade in flooded manholes, -to pull the red-eyed graveyard-shift over collapsing switching-systems. -The Bell ethic was the electrical equivalent of the postman's: -neither rain, nor snow, nor gloom of night would stop these couriers. - -It is easy to be cynical about this, as it is easy to be -cynical about any political or social system; but cynicism -does not change the fact that thousands of people took -these ideals very seriously. And some still do. - -The Bell ethos was about public service; and that was -gratifying; but it was also about private POWER, and that -was gratifying too. As a corporation, Bell was very special. -Bell was privileged. Bell had snuggled up close to the state. -In fact, Bell was as close to government as you could get in -America and still make a whole lot of legitimate money. - -But unlike other companies, Bell was above and beyond -the vulgar commercial fray. Through its regional operating companies, -Bell was omnipresent, local, and intimate, all over America; -but the central ivory towers at its corporate heart were the -tallest and the ivoriest around. - -There were other phone companies in America, to be sure; -the so-called independents. Rural cooperatives, mostly; -small fry, mostly tolerated, sometimes warred upon. -For many decades, "independent" American phone companies -lived in fear and loathing of the official Bell monopoly -(or the "Bell Octopus," as Ma Bell's nineteenth-century -enemies described her in many angry newspaper manifestos). -Some few of these independent entrepreneurs, while legally -in the wrong, fought so bitterly against the Octopus -that their illegal phone networks were cast into the street -by Bell agents and publicly burned. - -The pure technical sweetness of the Bell System gave its operators, -inventors and engineers a deeply satisfying sense of power and mastery. -They had devoted their lives to improving this vast nation-spanning machine; -over years, whole human lives, they had watched it improve and grow. -It was like a great technological temple. They were an elite, -and they knew it--even if others did not; in fact, they felt -even more powerful BECAUSE others did not understand. - -The deep attraction of this sensation of elite technical power -should never be underestimated. "Technical power" is not for everybody; -for many people it simply has no charm at all. But for some people, -it becomes the core of their lives. For a few, it is overwhelming, -obsessive; it becomes something close to an addiction. People--especially -clever teenage boys whose lives are otherwise mostly powerless and put-upon ---love this sensation of secret power, and are willing to do all sorts -of amazing things to achieve it. The technical POWER of electronics -has motivated many strange acts detailed in this book, which would -otherwise be inexplicable. - -So Bell had power beyond mere capitalism. The Bell service ethos worked, -and was often propagandized, in a rather saccharine fashion. Over the decades, -people slowly grew tired of this. And then, openly impatient with it. -By the early 1980s, Ma Bell was to find herself with scarcely a real friend -in the world. Vail's industrial socialism had become hopelessly -out-of-fashion politically. Bell would be punished for that. -And that punishment would fall harshly upon the people of the -telephone community. - -# - -In 1983, Ma Bell was dismantled by federal court action. -The pieces of Bell are now separate corporate entities. -The core of the company became AT&T Communications, -and also AT&T Industries (formerly Western Electric, -Bell's manufacturing arm). AT&T Bell Labs became Bell -Communications Research, Bellcore. Then there are the -Regional Bell Operating Companies, or RBOCs, pronounced "arbocks." - -Bell was a titan and even these regional chunks are gigantic enterprises: -Fortune 50 companies with plenty of wealth and power behind them. -But the clean lines of "One Policy, One System, Universal Service" -have been shattered, apparently forever. - -The "One Policy" of the early Reagan Administration was to -shatter a system that smacked of noncompetitive socialism. -Since that time, there has been no real telephone "policy" -on the federal level. Despite the breakup, the remnants -of Bell have never been set free to compete in the open marketplace. - -The RBOCs are still very heavily regulated, but not from the top. -Instead, they struggle politically, economically and legally, -in what seems an endless turmoil, in a patchwork of overlapping federal -and state jurisdictions. Increasingly, like other major American corporations, -the RBOCs are becoming multinational, acquiring important commercial interests -in Europe, Latin America, and the Pacific Rim. But this, too, adds to their -legal and political predicament. - -The people of what used to be Ma Bell are not happy about their fate. -They feel ill-used. They might have been grudgingly willing to make -a full transition to the free market; to become just companies amid -other companies. But this never happened. Instead, AT&T and the RBOCS -("the Baby Bells") feel themselves wrenched from side to side by state -regulators, by Congress, by the FCC, and especially by the federal court -of Judge Harold Greene, the magistrate who ordered the Bell breakup -and who has been the de facto czar of American telecommunications -ever since 1983. - -Bell people feel that they exist in a kind of paralegal limbo today. -They don't understand what's demanded of them. If it's "service," -why aren't they treated like a public service? And if it's money, -then why aren't they free to compete for it? No one seems to know, -really. Those who claim to know keep changing their minds. -Nobody in authority seems willing to grasp the nettle for once and all. - -Telephone people from other countries are amazed by the -American telephone system today. Not that it works so well; -for nowadays even the French telephone system works, more or less. -They are amazed that the American telephone system STILL works -AT ALL, under these strange conditions. - -Bell's "One System" of long-distance service is now only about -eighty percent of a system, with the remainder held by Sprint, MCI, -and the midget long-distance companies. Ugly wars over dubious -corporate practices such as "slamming" (an underhanded method -of snitching clients from rivals) break out with some regularity -in the realm of long-distance service. The battle to break Bell's -long-distance monopoly was long and ugly, and since the breakup -the battlefield has not become much prettier. AT&T's famous -shame-and-blame advertisements, which emphasized the shoddy work -and purported ethical shadiness of their competitors, were much -remarked on for their studied psychological cruelty. - -There is much bad blood in this industry, and much -long-treasured resentment. AT&T's post-breakup -corporate logo, a striped sphere, is known in the -industry as the "Death Star" (a reference from the movie -Star Wars, in which the "Death Star" was the spherical -high- tech fortress of the harsh-breathing imperial ultra-baddie, -Darth Vader.) Even AT&T employees are less than thrilled -by the Death Star. A popular (though banned) T-shirt among -AT&T employees bears the old-fashioned Bell logo of the Bell System, -plus the newfangled striped sphere, with the before-and-after comments: -"This is your brain--This is your brain on drugs!" AT&T made a very -well-financed and determined effort to break into the personal -computer market; it was disastrous, and telco computer experts -are derisively known by their competitors as "the pole-climbers." -AT&T and the Baby Bell arbocks still seem to have few friends. - -Under conditions of sharp commercial competition, a crash like -that of January 15, 1990 was a major embarrassment to AT&T. -It was a direct blow against their much-treasured reputation -for reliability. Within days of the crash AT&T's -Chief Executive Officer, Bob Allen, officially apologized, -in terms of deeply pained humility: - -"AT&T had a major service disruption last Monday. -We didn't live up to our own standards of quality, -and we didn't live up to yours. It's as simple as that. -And that's not acceptable to us. Or to you. . . . -We understand how much people have come to depend -upon AT&T service, so our AT&T Bell Laboratories scientists -and our network engineers are doing everything possible -to guard against a recurrence. . . . We know there's no way -to make up for the inconvenience this problem may have caused you." - -Mr Allen's "open letter to customers" was printed in lavish ads -all over the country: in the Wall Street Journal, USA Today, -New York Times, Los Angeles Times, Chicago Tribune, -Philadelphia Inquirer, San Francisco Chronicle Examiner, -Boston Globe, Dallas Morning News, Detroit Free Press, -Washington Post, Houston Chronicle, Cleveland Plain Dealer, -Atlanta Journal Constitution, Minneapolis Star Tribune, -St. Paul Pioneer Press Dispatch, Seattle Times/Post Intelligencer, -Tacoma News Tribune, Miami Herald, Pittsburgh Press, -St. Louis Post Dispatch, Denver Post, Phoenix Republic Gazette -and Tampa Tribune. - -In another press release, AT&T went to some pains to suggest -that this "software glitch" might have happened just as easily to MCI, -although, in fact, it hadn't. (MCI's switching software was quite different -from AT&T's--though not necessarily any safer.) AT&T also announced -their plans to offer a rebate of service on Valentine's Day to make up -for the loss during the Crash. - -"Every technical resource available, including Bell Labs -scientists and engineers, has been devoted to assuring -it will not occur again," the public was told. They were -further assured that "The chances of a recurrence are small-- -a problem of this magnitude never occurred before." - -In the meantime, however, police and corporate -security maintained their own suspicions about -"the chances of recurrence" and the real reason why -a "problem of this magnitude" had appeared, seemingly -out of nowhere. Police and security knew for a fact -that hackers of unprecedented sophistication were illegally -entering, and reprogramming, certain digital switching stations. -Rumors of hidden "viruses" and secret "logic bombs" -in the switches ran rampant in the underground, -with much chortling over AT&T's predicament, -and idle speculation over what unsung hacker genius -was responsible for it. Some hackers, including police -informants, were trying hard to finger one another -as the true culprits of the Crash. - -Telco people found little comfort in objectivity when -they contemplated these possibilities. It was just too close -to the bone for them; it was embarrassing; it hurt so much, -it was hard even to talk about. - -There has always been thieving and misbehavior in the phone system. -There has always been trouble with the rival independents, -and in the local loops. But to have such trouble in the core -of the system, the long-distance switching stations, -is a horrifying affair. To telco people, this is -all the difference between finding roaches in your kitchen -and big horrid sewer-rats in your bedroom. - -From the outside, to the average citizen, the telcos -still seem gigantic and impersonal. The American public -seems to regard them as something akin to Soviet apparats. -Even when the telcos do their best corporate-citizen routine, -subsidizing magnet high-schools and sponsoring news-shows -on public television, they seem to win little except public suspicion. - -But from the inside, all this looks very different. -There's harsh competition. A legal and political system -that seems baffled and bored, when not actively hostile -to telco interests. There's a loss of morale, a deep sensation -of having somehow lost the upper hand. Technological change -has caused a loss of data and revenue to other, newer forms -of transmission. There's theft, and new forms of theft, -of growing scale and boldness and sophistication. -With all these factors, it was no surprise to see the telcos, -large and small, break out in a litany of bitter complaint. - -In late '88 and throughout 1989, telco representatives -grew shrill in their complaints to those few American law -enforcement officials who make it their business to try to -understand what telephone people are talking about. -Telco security officials had discovered the computer- -hacker underground, infiltrated it thoroughly, -and become deeply alarmed at its growing expertise. -Here they had found a target that was not only loathsome -on its face, but clearly ripe for counterattack. - -Those bitter rivals: AT&T, MCI and Sprint--and a crowd -of Baby Bells: PacBell, Bell South, Southwestern Bell, -NYNEX, USWest, as well as the Bell research consortium Bellcore, -and the independent long-distance carrier Mid-American-- -all were to have their role in the great hacker dragnet of 1990. -After years of being battered and pushed around, the telcos had, -at least in a small way, seized the initiative again. -After years of turmoil, telcos and government officials were -once again to work smoothly in concert in defense of the System. -Optimism blossomed; enthusiasm grew on all sides; -the prospective taste of vengeance was sweet. - -# - -From the beginning--even before the crackdown had a name-- -secrecy was a big problem. There were many good reasons -for secrecy in the hacker crackdown. Hackers and code-thieves -were wily prey, slinking back to their bedrooms and basements -and destroying vital incriminating evidence at the first hint of trouble. -Furthermore, the crimes themselves were heavily technical and difficult -to describe, even to police--much less to the general public. - -When such crimes HAD been described intelligibly to the public, -in the past, that very publicity had tended to INCREASE the crimes -enormously. Telco officials, while painfully aware of the vulnerabilities -of their systems, were anxious not to publicize those weaknesses. -Experience showed them that those weaknesses, once discovered, -would be pitilessly exploited by tens of thousands of people--not only -by professional grifters and by underground hackers and phone phreaks, -but by many otherwise more-or-less honest everyday folks, who regarded -stealing service from the faceless, soulless "Phone Company" as a kind of -harmless indoor sport. When it came to protecting their interests, -telcos had long since given up on general public sympathy for -"the Voice with a Smile." Nowadays the telco's "Voice" was -very likely to be a computer's; and the American public -showed much less of the proper respect and gratitude due -the fine public service bequeathed them by Dr. Bell and Mr. Vail. -The more efficient, high-tech, computerized, and impersonal -the telcos became, it seemed, the more they were met by -sullen public resentment and amoral greed. - -Telco officials wanted to punish the phone-phreak underground, in as -public and exemplary a manner as possible. They wanted to make dire -examples of the worst offenders, to seize the ringleaders and intimidate -the small fry, to discourage and frighten the wacky hobbyists, and send -the professional grifters to jail. To do all this, publicity was vital. - -Yet operational secrecy was even more so. If word got out that -a nationwide crackdown was coming, the hackers might simply vanish; -destroy the evidence, hide their computers, go to earth, -and wait for the campaign to blow over. Even the young -hackers were crafty and suspicious, and as for the professional grifters, -they tended to split for the nearest state-line at the first sign of trouble. -For the crackdown to work well, they would all have to be caught red-handed, -swept upon suddenly, out of the blue, from every corner of the compass. - -And there was another strong motive for secrecy. In the worst-case scenario, -a blown campaign might leave the telcos open to a devastating hacker -counter-attack. If there were indeed hackers loose in America who -had caused the January 15 Crash--if there were truly gifted hackers, -loose in the nation's long-distance switching systems, and enraged -or frightened by the crackdown--then they might react unpredictably -to an attempt to collar them. Even if caught, they might have talented -and vengeful friends still running around loose. Conceivably, -it could turn ugly. Very ugly. In fact, it was hard to imagine -just how ugly things might turn, given that possibility. - -Counter-attack from hackers was a genuine concern for the telcos. -In point of fact, they would never suffer any such counter-attack. -But in months to come, they would be at some pains to publicize -this notion and to utter grim warnings about it. - -Still, that risk seemed well worth running. Better to run the risk -of vengeful attacks, than to live at the mercy of potential crashers. -Any cop would tell you that a protection racket had no real future. - -And publicity was such a useful thing. Corporate security officers, -including telco security, generally work under conditions of great discretion. -And corporate security officials do not make money for their companies. -Their job is to PREVENT THE LOSS of money, which is much less glamorous -than actually winning profits. - -If you are a corporate security official, and you do your job brilliantly, -then nothing bad happens to your company at all. Because of this, you appear -completely superfluous. This is one of the many unattractive aspects -of security work. It's rare that these folks have the chance to draw -some healthy attention to their own efforts. - -Publicity also served the interest of their friends in law enforcement. -Public officials, including law enforcement officials, thrive by attracting -favorable public interest. A brilliant prosecution in a matter of vital -public interest can make the career of a prosecuting attorney. -And for a police officer, good publicity opens the purses of the legislature; -it may bring a citation, or a promotion, or at least a rise in status -and the respect of one's peers. - -But to have both publicity and secrecy is to have one's cake and eat it too. -In months to come, as we will show, this impossible act was to cause great -pain to the agents of the crackdown. But early on, it seemed possible ---maybe even likely--that the crackdown could successfully combine -the best of both worlds. The ARREST of hackers would be heavily publicized. -The actual DEEDS of the hackers, which were technically hard to explain -and also a security risk, would be left decently obscured. The THREAT -hackers posed would be heavily trumpeted; the likelihood of their actually -committing such fearsome crimes would be left to the public's imagination. -The spread of the computer underground, and its growing technical -sophistication, would be heavily promoted; the actual hackers themselves, -mostly bespectacled middle-class white suburban teenagers, -would be denied any personal publicity. - -It does not seem to have occurred to any telco official -that the hackers accused would demand a day in court; -that journalists would smile upon the hackers as -"good copy;" that wealthy high-tech entrepreneurs would -offer moral and financial support to crackdown victims; -that constitutional lawyers would show up with briefcases, -frowning mightily. This possibility does not seem to have -ever entered the game-plan. - -And even if it had, it probably would not have slowed -the ferocious pursuit of a stolen phone-company document, -mellifluously known as "Control Office Administration of -Enhanced 911 Services for Special Services and Major Account Centers." - -In the chapters to follow, we will explore the worlds -of police and the computer underground, and the large -shadowy area where they overlap. But first, we must -explore the battleground. Before we leave the world -of the telcos, we must understand what a switching system -actually is and how your telephone actually works. - -# - -To the average citizen, the idea of the telephone is represented by, -well, a TELEPHONE: a device that you talk into. To a telco -professional, however, the telephone itself is known, in lordly -fashion, as a "subset." The "subset" in your house is a mere adjunct, -a distant nerve ending, of the central switching stations, -which are ranked in levels of heirarchy, up to the long-distance electronic -switching stations, which are some of the largest computers on earth. - -Let us imagine that it is, say, 1925, before the -introduction of computers, when the phone system was -simpler and somewhat easier to grasp. Let's further -imagine that you are Miss Leticia Luthor, a fictional -operator for Ma Bell in New York City of the 20s. - -Basically, you, Miss Luthor, ARE the "switching system." -You are sitting in front of a large vertical switchboard, -known as a "cordboard," made of shiny wooden panels, -with ten thousand metal-rimmed holes punched in them, -known as jacks. The engineers would have put more -holes into your switchboard, but ten thousand is -as many as you can reach without actually having -to get up out of your chair. - -Each of these ten thousand holes has its own little electric lightbulb, -known as a "lamp," and its own neatly printed number code. - -With the ease of long habit, you are scanning your board for lit-up bulbs. -This is what you do most of the time, so you are used to it. - -A lamp lights up. This means that the phone -at the end of that line has been taken off the hook. -Whenever a handset is taken off the hook, that closes a circuit -inside the phone which then signals the local office, i.e. you, -automatically. There might be somebody calling, or then -again the phone might be simply off the hook, but this -does not matter to you yet. The first thing you do, -is record that number in your logbook, in your fine American -public-school handwriting. This comes first, naturally, -since it is done for billing purposes. - -You now take the plug of your answering cord, which goes -directly to your headset, and plug it into the lit-up hole. -"Operator," you announce. - -In operator's classes, before taking this job, you have -been issued a large pamphlet full of canned operator's -responses for all kinds of contingencies, which you had -to memorize. You have also been trained in a proper -non-regional, non-ethnic pronunciation and tone of voice. -You rarely have the occasion to make any spontaneous -remark to a customer, and in fact this is frowned upon -(except out on the rural lines where people have time -on their hands and get up to all kinds of mischief). - -A tough-sounding user's voice at the end of the line -gives you a number. Immediately, you write that number -down in your logbook, next to the caller's number, -which you just wrote earlier. You then look and see if -the number this guy wants is in fact on your switchboard, -which it generally is, since it's generally a local call. -Long distance costs so much that people use it sparingly. - -Only then do you pick up a calling-cord from a shelf -at the base of the switchboard. This is a long elastic cord -mounted on a kind of reel so that it will zip back in when -you unplug it. There are a lot of cords down there, -and when a bunch of them are out at once they look like -a nest of snakes. Some of the girls think there are bugs -living in those cable-holes. They're called "cable mites" -and are supposed to bite your hands and give you rashes. -You don't believe this, yourself. - -Gripping the head of your calling-cord, you slip the tip -of it deftly into the sleeve of the jack for the called person. -Not all the way in, though. You just touch it. If you hear -a clicking sound, that means the line is busy and you can't -put the call through. If the line is busy, you have to stick -the calling-cord into a "busy-tone jack," which will give -the guy a busy-tone. This way you don't have to talk to him -yourself and absorb his natural human frustration. - -But the line isn't busy. So you pop the cord all the way in. -Relay circuits in your board make the distant phone ring, -and if somebody picks it up off the hook, then a phone -conversation starts. You can hear this conversation -on your answering cord, until you unplug it. In fact -you could listen to the whole conversation if you wanted, -but this is sternly frowned upon by management, and frankly, -when you've overheard one, you've pretty much heard 'em all. - -You can tell how long the conversation lasts by the glow -of the calling-cord's lamp, down on the calling-cord's shelf. -When it's over, you unplug and the calling-cord zips back into place. - -Having done this stuff a few hundred thousand times, -you become quite good at it. In fact you're plugging, -and connecting, and disconnecting, ten, twenty, forty cords -at a time. It's a manual handicraft, really, quite satisfying -in a way, rather like weaving on an upright loom. - -Should a long-distance call come up, it would be different, -but not all that different. Instead of connecting the call -through your own local switchboard, you have to go up the hierarchy, -onto the long-distance lines, known as "trunklines." -Depending on how far the call goes, it may have to work -its way through a whole series of operators, which can -take quite a while. The caller doesn't wait on the line -while this complex process is negotiated across the country -by the gaggle of operators. Instead, the caller hangs up, -and you call him back yourself when the call has finally -worked its way through. - -After four or five years of this work, you get married, -and you have to quit your job, this being the natural order -of womanhood in the American 1920s. The phone company -has to train somebody else--maybe two people, since -the phone system has grown somewhat in the meantime. -And this costs money. - -In fact, to use any kind of human being as a switching -system is a very expensive proposition. Eight thousand -Leticia Luthors would be bad enough, but a quarter of a -million of them is a military-scale proposition and makes -drastic measures in automation financially worthwhile. - -Although the phone system continues to grow today, -the number of human beings employed by telcos has -been dropping steadily for years. Phone "operators" -now deal with nothing but unusual contingencies, -all routine operations having been shrugged off onto machines. -Consequently, telephone operators are considerably less -machine-like nowadays, and have been known to have accents -and actual character in their voices. When you reach -a human operator today, the operators are rather more -"human" than they were in Leticia's day--but on the other hand, -human beings in the phone system are much harder to reach -in the first place. - -Over the first half of the twentieth century, -"electromechanical" switching systems of growing -complexity were cautiously introduced into the phone system. -In certain backwaters, some of these hybrid systems are still -in use. But after 1965, the phone system began to go completely -electronic, and this is by far the dominant mode today. -Electromechanical systems have "crossbars," and "brushes," -and other large moving mechanical parts, which, while faster -and cheaper than Leticia, are still slow, and tend to wear out -fairly quickly. - -But fully electronic systems are inscribed on silicon chips, -and are lightning-fast, very cheap, and quite durable. -They are much cheaper to maintain than even the best -electromechanical systems, and they fit into half the space. -And with every year, the silicon chip grows smaller, faster, -and cheaper yet. Best of all, automated electronics work -around the clock and don't have salaries or health insurance. - -There are, however, quite serious drawbacks to the -use of computer-chips. When they do break down, it is -a daunting challenge to figure out what the heck has gone -wrong with them. A broken cordboard generally had -a problem in it big enough to see. A broken chip has -invisible, microscopic faults. And the faults in bad -software can be so subtle as to be practically theological. - -If you want a mechanical system to do something new, -then you must travel to where it is, and pull pieces out of it, -and wire in new pieces. This costs money. However, if you want -a chip to do something new, all you have to do is change its software, -which is easy, fast and dirt-cheap. You don't even have to see the chip -to change its program. Even if you did see the chip, it wouldn't look -like much. A chip with program X doesn't look one whit different from -a chip with program Y. - -With the proper codes and sequences, and access to specialized phone-lines, -you can change electronic switching systems all over America from anywhere -you please. - -And so can other people. If they know how, and if they want to, -they can sneak into a microchip via the special phonelines and diddle with it, -leaving no physical trace at all. If they broke into the operator's station -and held Leticia at gunpoint, that would be very obvious. If they broke into -a telco building and went after an electromechanical switch with a toolbelt, -that would at least leave many traces. But people can do all manner of amazing -things to computer switches just by typing on a keyboard, and keyboards are -everywhere today. The extent of this vulnerability is deep, dark, broad, -almost mind-boggling, and yet this is a basic, primal fact of life about -any computer on a network. - -Security experts over the past twenty years have insisted, -with growing urgency, that this basic vulnerability of computers -represents an entirely new level of risk, of unknown but obviously -dire potential to society. And they are right. - -An electronic switching station does pretty much -everything Letitia did, except in nanoseconds and -on a much larger scale. Compared to Miss Luthor's -ten thousand jacks, even a primitive 1ESS switching computer, -60s vintage, has a 128,000 lines. And the current AT&T -system of choice is the monstrous fifth-generation 5ESS. - -An Electronic Switching Station can scan every line on its "board" -in a tenth of a second, and it does this over and over, tirelessly, -around the clock. Instead of eyes, it uses "ferrod scanners" -to check the condition of local lines and trunks. Instead of hands, -it has "signal distributors," "central pulse distributors," -"magnetic latching relays," and "reed switches," which complete -and break the calls. Instead of a brain, it has a "central processor." -Instead of an instruction manual, it has a program. Instead of -a handwritten logbook for recording and billing calls, -it has magnetic tapes. And it never has to talk to anybody. -Everything a customer might say to it is done by punching -the direct-dial tone buttons on your subset. - -Although an Electronic Switching Station can't talk, -it does need an interface, some way to relate to its, er, -employers. This interface is known as the "master control -center." (This interface might be better known simply as -"the interface," since it doesn't actually "control" phone -calls directly. However, a term like "Master Control -Center" is just the kind of rhetoric that telco maintenance -engineers--and hackers--find particularly satisfying.) - -Using the master control center, a phone engineer can test -local and trunk lines for malfunctions. He (rarely she) -can check various alarm displays, measure traffic on the lines, -examine the records of telephone usage and the charges for those calls, -and change the programming. - -And, of course, anybody else who gets into the master control center -by remote control can also do these things, if he (rarely she) -has managed to figure them out, or, more likely, has somehow swiped -the knowledge from people who already know. - -In 1989 and 1990, one particular RBOC, BellSouth, -which felt particularly troubled, spent a purported $1.2 -million on computer security. Some think it spent as -much as two million, if you count all the associated costs. -Two million dollars is still very little compared to the -great cost-saving utility of telephonic computer systems. - -Unfortunately, computers are also stupid. -Unlike human beings, computers possess the truly -profound stupidity of the inanimate. - -In the 1960s, in the first shocks of spreading computerization, -there was much easy talk about the stupidity of computers-- -how they could "only follow the program" and were rigidly required -to do "only what they were told." There has been rather less talk -about the stupidity of computers since they began to achieve -grandmaster status in chess tournaments, and to manifest -many other impressive forms of apparent cleverness. - -Nevertheless, computers STILL are profoundly brittle and stupid; -they are simply vastly more subtle in their stupidity and brittleness. -The computers of the 1990s are much more reliable in their components -than earlier computer systems, but they are also called upon to do -far more complex things, under far more challenging conditions. - -On a basic mathematical level, every single line of -a software program offers a chance for some possible screwup. -Software does not sit still when it works; it "runs," -it interacts with itself and with its own inputs and outputs. -By analogy, it stretches like putty into millions of possible -shapes and conditions, so many shapes that they can never -all be successfully tested, not even in the lifespan of the universe. -Sometimes the putty snaps. - -The stuff we call "software" is not like anything that human society -is used to thinking about. Software is something like a machine, -and something like mathematics, and something like language, and -something like thought, and art, and information. . . . But software -is not in fact any of those other things. The protean quality -of software is one of the great sources of its fascination. -It also makes software very powerful, very subtle, -very unpredictable, and very risky. - -Some software is bad and buggy. Some is "robust," -even "bulletproof." The best software is that which has -been tested by thousands of users under thousands of -different conditions, over years. It is then known as -"stable." This does NOT mean that the software is -now flawless, free of bugs. It generally means that there -are plenty of bugs in it, but the bugs are well-identified -and fairly well understood. - -There is simply no way to assure that software is free -of flaws. Though software is mathematical in nature, -it cannot by "proven" like a mathematical theorem; -software is more like language, with inherent ambiguities, -with different definitions, different assumptions, -different levels of meaning that can conflict. - -Human beings can manage, more or less, with -human language because we can catch the gist of it. - -Computers, despite years of effort in "artificial intelligence," -have proven spectacularly bad in "catching the gist" of anything at all. -The tiniest bit of semantic grit may still bring the mightiest computer -tumbling down. One of the most hazardous things you can do to a -computer program is try to improve it--to try to make it safer. -Software "patches" represent new, untried un-"stable" software, -which is by definition riskier. - -The modern telephone system has come to depend, -utterly and irretrievably, upon software. And the -System Crash of January 15, 1990, was caused by an -IMPROVEMENT in software. Or rather, an ATTEMPTED -improvement. - -As it happened, the problem itself--the problem per se--took this form. -A piece of telco software had been written in C language, a standard -language of the telco field. Within the C software was a -long "do. . .while" construct. The "do. . .while" construct -contained a "switch" statement. The "switch" statement contained -an "if" clause. The "if" clause contained a "break." The "break" -was SUPPOSED to "break" the "if clause." Instead, the "break" -broke the "switch" statement. - -That was the problem, the actual reason why people picking up phones -on January 15, 1990, could not talk to one another. - -Or at least, that was the subtle, abstract, cyberspatial -seed of the problem. This is how the problem manifested itself -from the realm of programming into the realm of real life. - -The System 7 software for AT&T's 4ESS switching station, -the "Generic 44E14 Central Office Switch Software," -had been extensively tested, and was considered very stable. -By the end of 1989, eighty of AT&T's switching systems -nationwide had been programmed with the new software. Cautiously, -thirty-four stations were left to run the slower, less-capable -System 6, because AT&T suspected there might be shakedown problems -with the new and unprecedently sophisticated System 7 network. - -The stations with System 7 were programmed to switch over to a backup net -in case of any problems. In mid-December 1989, however, a new high-velocity, -high-security software patch was distributed to each of the 4ESS switches -that would enable them to switch over even more quickly, making the System 7 -network that much more secure. - -Unfortunately, every one of these 4ESS switches was now in possession -of a small but deadly flaw. - -In order to maintain the network, switches must monitor -the condition of other switches--whether they are up and running, -whether they have temporarily shut down, whether they are overloaded -and in need of assistance, and so forth. The new software helped -control this bookkeeping function by monitoring the status calls -from other switches. - -It only takes four to six seconds for a troubled 4ESS switch -to rid itself of all its calls, drop everything temporarily, -and re-boot its software from scratch. Starting over from scratch -will generally rid the switch of any software problems that may have -developed in the course of running the system. Bugs that arise will -be simply wiped out by this process. It is a clever idea. This process -of automatically re-booting from scratch is known as the "normal fault -recovery routine." Since AT&T's software is in fact exceptionally stable, -systems rarely have to go into "fault recovery" in the first place; -but AT&T has always boasted of its "real world" reliability, and this -tactic is a belt-and-suspenders routine. - -The 4ESS switch used its new software to monitor its fellow switches -as they recovered from faults. As other switches came back on line -after recovery, they would send their "OK" signals to the switch. -The switch would make a little note to that effect in its "status map," -recognizing that the fellow switch was back and ready to go, -and should be sent some calls and put back to regular work. - -Unfortunately, while it was busy bookkeeping with the status map, -the tiny flaw in the brand-new software came into play. -The flaw caused the 4ESS switch to interact, subtly but drastically, -with incoming telephone calls from human users. If--and only if-- -two incoming phone-calls happened to hit the switch within a hundredth -of a second, then a small patch of data would be garbled by the flaw. - -But the switch had been programmed to monitor itself -constantly for any possible damage to its data. -When the switch perceived that its data had been somehow garbled, -then it too would go down, for swift repairs to its software. -It would signal its fellow switches not to send any more work. -It would go into the fault-recovery mode for four to six seconds. -And then the switch would be fine again, and would send out its "OK, -ready for work" signal. - -However, the "OK, ready for work" signal was the VERY THING THAT -HAD CAUSED THE SWITCH TO GO DOWN IN THE FIRST PLACE. And ALL the -System 7 switches had the same flaw in their status-map software. -As soon as they stopped to make the bookkeeping note that their fellow -switch was "OK," then they too would become vulnerable to the slight -chance that two phone-calls would hit them within a hundredth of a second. - -At approximately 2:25 P.M. EST on Monday, January 15, -one of AT&T's 4ESS toll switching systems in New York City -had an actual, legitimate, minor problem. It went into fault -recovery routines, announced "I'm going down," then announced, -"I'm back, I'm OK." And this cheery message then blasted -throughout the network to many of its fellow 4ESS switches. - -Many of the switches, at first, completely escaped trouble. -These lucky switches were not hit by the coincidence of -two phone calls within a hundredth of a second. -Their software did not fail--at first. But three switches-- -in Atlanta, St. Louis, and Detroit--were unlucky, -and were caught with their hands full. And they went down. -And they came back up, almost immediately. And they too began -to broadcast the lethal message that they, too, were "OK" again, -activating the lurking software bug in yet other switches. - -As more and more switches did have that bit of bad luck -and collapsed, the call-traffic became more and more densely -packed in the remaining switches, which were groaning -to keep up with the load. And of course, as the calls -became more densely packed, the switches were MUCH MORE LIKELY -to be hit twice within a hundredth of a second. - -It only took four seconds for a switch to get well. -There was no PHYSICAL damage of any kind to the switches, -after all. Physically, they were working perfectly. -This situation was "only" a software problem. - -But the 4ESS switches were leaping up and down every -four to six seconds, in a virulent spreading wave all over America, -in utter, manic, mechanical stupidity. They kept KNOCKING -one another down with their contagious "OK" messages. - -It took about ten minutes for the chain reaction to cripple the network. -Even then, switches would periodically luck-out and manage to resume -their normal work. Many calls--millions of them--were managing -to get through. But millions weren't. - -The switching stations that used System 6 were not directly affected. -Thanks to these old-fashioned switches, AT&T's national system avoided -complete collapse. This fact also made it clear to engineers that -System 7 was at fault. - -Bell Labs engineers, working feverishly in New Jersey, Illinois, -and Ohio, first tried their entire repertoire of standard network -remedies on the malfunctioning System 7. None of the remedies worked, -of course, because nothing like this had ever happened to any -phone system before. - -By cutting out the backup safety network entirely, -they were able to reduce the frenzy of "OK" messages -by about half. The system then began to recover, as the -chain reaction slowed. By 11:30 P.M. on Monday January -15, sweating engineers on the midnight shift breathed a -sigh of relief as the last switch cleared-up. - -By Tuesday they were pulling all the brand-new 4ESS software -and replacing it with an earlier version of System 7. - -If these had been human operators, rather than -computers at work, someone would simply have -eventually stopped screaming. It would have been -OBVIOUS that the situation was not "OK," and common -sense would have kicked in. Humans possess common sense-- -at least to some extent. Computers simply don't. - -On the other hand, computers can handle hundreds -of calls per second. Humans simply can't. If every single -human being in America worked for the phone company, -we couldn't match the performance of digital switches: -direct-dialling, three-way calling, speed-calling, call- -waiting, Caller ID, all the rest of the cornucopia -of digital bounty. Replacing computers with operators -is simply not an option any more. - -And yet we still, anachronistically, expect humans to -be running our phone system. It is hard for us -to understand that we have sacrificed huge amounts -of initiative and control to senseless yet powerful machines. -When the phones fail, we want somebody to be responsible. -We want somebody to blame. - -When the Crash of January 15 happened, the American populace -was simply not prepared to understand that enormous landslides -in cyberspace, like the Crash itself, can happen, -and can be nobody's fault in particular. It was easier to believe, -maybe even in some odd way more reassuring to believe, -that some evil person, or evil group, had done this to us. -"Hackers" had done it. With a virus. A trojan horse. -A software bomb. A dirty plot of some kind. People believed this, -responsible people. In 1990, they were looking hard for evidence -to confirm their heartfelt suspicions. - -And they would look in a lot of places. - -Come 1991, however, the outlines of an apparent new reality -would begin to emerge from the fog. - -On July 1 and 2, 1991, computer-software collapses -in telephone switching stations disrupted service in -Washington DC, Pittsburgh, Los Angeles and San Francisco. -Once again, seemingly minor maintenance problems had -crippled the digital System 7. About twelve million -people were affected in the Crash of July 1, 1991. - -Said the New York Times Service: "Telephone company executives -and federal regulators said they were not ruling out the possibility -of sabotage by computer hackers, but most seemed to think the problems -stemmed from some unknown defect in the software running the networks." - -And sure enough, within the week, a red-faced software company, -DSC Communications Corporation of Plano, Texas, owned up -to "glitches" in the "signal transfer point" software that -DSC had designed for Bell Atlantic and Pacific Bell. -The immediate cause of the July 1 Crash was a single -mistyped character: one tiny typographical flaw -in one single line of the software. One mistyped letter, -in one single line, had deprived the nation's capital of phone service. -It was not particularly surprising that this tiny flaw had escaped attention: -a typical System 7 station requires TEN MILLION lines of code. - -On Tuesday, September 17, 1991, came the most spectacular outage yet. -This case had nothing to do with software failures--at least, not directly. -Instead, a group of AT&T's switching stations in New York City had simply -run out of electrical power and shut down cold. Their back-up batteries -had failed. Automatic warning systems were supposed to warn of the loss -of battery power, but those automatic systems had failed as well. - -This time, Kennedy, La Guardia, and Newark airports -all had their voice and data communications cut. -This horrifying event was particularly ironic, as attacks -on airport computers by hackers had long been a standard -nightmare scenario, much trumpeted by computer-security -experts who feared the computer underground. There had even -been a Hollywood thriller about sinister hackers ruining -airport computers--DIE HARD II. - -Now AT&T itself had crippled airports with computer malfunctions-- -not just one airport, but three at once, some of the busiest in the world. - -Air traffic came to a standstill throughout the Greater New York area, -causing more than 500 flights to be cancelled, in a spreading wave -all over America and even into Europe. Another 500 or so flights -were delayed, affecting, all in all, about 85,000 passengers. -(One of these passengers was the chairman of the Federal -Communications Commission.) - -Stranded passengers in New York and New Jersey were further -infuriated to discover that they could not even manage to -make a long distance phone call, to explain their delay -to loved ones or business associates. Thanks to the crash, -about four and a half million domestic calls, and half a million -international calls, failed to get through. - -The September 17 NYC Crash, unlike the previous ones, -involved not a whisper of "hacker" misdeeds. On the contrary, -by 1991, AT&T itself was suffering much of the vilification -that had formerly been directed at hackers. Congressmen were grumbling. -So were state and federal regulators. And so was the press. - -For their part, ancient rival MCI took out snide full-page -newspaper ads in New York, offering their own long-distance -services for the "next time that AT&T goes down." - -"You wouldn't find a classy company like AT&T using such advertising," -protested AT&T Chairman Robert Allen, unconvincingly. Once again, -out came the full-page AT&T apologies in newspapers, apologies for -"an inexcusable culmination of both human and mechanical failure." -(This time, however, AT&T offered no discount on later calls. -Unkind critics suggested that AT&T were worried about setting any precedent -for refunding the financial losses caused by telephone crashes.) - -Industry journals asked publicly if AT&T was "asleep at the switch." -The telephone network, America's purported marvel of high-tech reliability, -had gone down three times in 18 months. Fortune magazine listed the -Crash of September 17 among the "Biggest Business Goofs of 1991," -cruelly parodying AT&T's ad campaign in an article entitled -"AT&T Wants You Back (Safely On the Ground, God Willing)." - -Why had those New York switching systems simply run out of power? -Because no human being had attended to the alarm system. -Why did the alarm systems blare automatically, -without any human being noticing? Because the three -telco technicians who SHOULD have been listening -were absent from their stations in the power-room, -on another floor of the building--attending a training class. -A training class about the alarm systems for the power room! - -"Crashing the System" was no longer "unprecedented" by late 1991. -On the contrary, it no longer even seemed an oddity. By 1991, -it was clear that all the policemen in the world could no longer -"protect" the phone system from crashes. By far the worst crashes -the system had ever had, had been inflicted, by the system, -upon ITSELF. And this time nobody was making cocksure statements -that this was an anomaly, something that would never happen again. -By 1991 the System's defenders had met their nebulous Enemy, -and the Enemy was--the System. - - - -PART TWO: THE DIGITAL UNDERGROUND - - -The date was May 9, 1990. The Pope was touring Mexico City. -Hustlers from the Medellin Cartel were trying to buy -black-market Stinger missiles in Florida. On the comics page, -Doonesbury character Andy was dying of AIDS. And then. . .a highly -unusual item whose novelty and calculated rhetoric won it -headscratching attention in newspapers all over America. - -The US Attorney's office in Phoenix, Arizona, had issued -a press release announcing a nationwide law enforcement crackdown -against "illegal computer hacking activities." The sweep was -officially known as "Operation Sundevil." - -Eight paragraphs in the press release gave the bare facts: -twenty-seven search warrants carried out on May 8, with three arrests, -and a hundred and fifty agents on the prowl in "twelve" cities across America. -(Different counts in local press reports yielded "thirteen," "fourteen," and -"sixteen" cities.) Officials estimated that criminal losses of revenue -to telephone companies "may run into millions of dollars." Credit for -the Sundevil investigations was taken by the US Secret Service, -Assistant US Attorney Tim Holtzen of Phoenix, and the Assistant -Attorney General of Arizona, Gail Thackeray. - -The prepared remarks of Garry M. Jenkins, appearing in a U.S. Department -of Justice press release, were of particular interest. Mr. Jenkins was the -Assistant Director of the US Secret Service, and the highest-ranking federal -official to take any direct public role in the hacker crackdown of 1990. - -"Today, the Secret Service is sending a clear message to those computer hackers -who have decided to violate the laws of this nation in the mistaken belief -that they can successfully avoid detection by hiding behind the relative -anonymity of their computer terminals. (. . .) "Underground groups have been -formed for the purpose of exchanging information relevant to their criminal -activities. These groups often communicate with each other through message -systems between computers called `bulletin boards.' "Our experience shows -that many computer hacker suspects are no longer misguided teenagers, -mischievously playing games with their computers in their bedrooms. -Some are now high tech computer operators using computers to engage -in unlawful conduct." - -Who were these "underground groups" and "high-tech operators?" -Where had they come from? What did they want? Who WERE they? -Were they "mischievous?" Were they dangerous? How had "misguided teenagers" -managed to alarm the United States Secret Service? And just how widespread -was this sort of thing? - -Of all the major players in the Hacker Crackdown: the phone companies, -law enforcement, the civil libertarians, and the "hackers" themselves-- -the "hackers" are by far the most mysterious, by far the hardest to -understand, by far the WEIRDEST. - -Not only are "hackers" novel in their activities, but they come -in a variety of odd subcultures, with a variety of languages, -motives and values. - -The earliest proto-hackers were probably those unsung mischievous -telegraph boys who were summarily fired by the Bell Company in 1878. - -Legitimate "hackers," those computer enthusiasts who are independent-minded -but law-abiding, generally trace their spiritual ancestry to elite technical -universities, especially M.I.T. and Stanford, in the 1960s. - -But the genuine roots of the modern hacker UNDERGROUND can probably be traced -most successfully to a now much-obscured hippie anarchist movement known as -the Yippies. The Yippies, who took their name from the largely fictional -"Youth International Party," carried out a loud and lively policy of surrealistic -subversion and outrageous political mischief. Their basic tenets were flagrant -sexual promiscuity, open and copious drug use, the political overthrow of any -powermonger over thirty years of age, and an immediate end to the war -in Vietnam, by any means necessary, including the psychic levitation -of the Pentagon. - -The two most visible Yippies were Abbie Hoffman and Jerry Rubin. -Rubin eventually became a Wall Street broker. Hoffman, ardently sought -by federal authorities, went into hiding for seven years, -in Mexico, France, and the United States. While on the lam, -Hoffman continued to write and publish, with help from sympathizers -in the American anarcho-leftist underground. Mostly, Hoffman survived -through false ID and odd jobs. Eventually he underwent facial plastic -surgery and adopted an entirely new identity as one "Barry Freed." -After surrendering himself to authorities in 1980, Hoffman spent a year -in prison on a cocaine conviction. - -Hoffman's worldview grew much darker as the glory days of the 1960s faded. -In 1989, he purportedly committed suicide, under odd and, to some, rather -suspicious circumstances. - -Abbie Hoffman is said to have caused the Federal Bureau of Investigation -to amass the single largest investigation file ever opened on an individual -American citizen. (If this is true, it is still questionable whether the -FBI regarded Abbie Hoffman a serious public threat--quite possibly, -his file was enormous simply because Hoffman left colorful legendry -wherever he went). He was a gifted publicist, who regarded electronic -media as both playground and weapon. He actively enjoyed manipulating -network TV and other gullible, image-hungry media, with various weird lies, -mindboggling rumors, impersonation scams, and other sinister distortions, -all absolutely guaranteed to upset cops, Presidential candidates, -and federal judges. Hoffman's most famous work was a book self-reflexively -known as STEAL THIS BOOK, which publicized a number of methods by which young, -penniless hippie agitators might live off the fat of a system supported by -humorless drones. STEAL THIS BOOK, whose title urged readers to damage -the very means of distribution which had put it into their hands, -might be described as a spiritual ancestor of a computer virus. - -Hoffman, like many a later conspirator, made extensive use of -pay-phones for his agitation work--in his case, generally through -the use of cheap brass washers as coin-slugs. - -During the Vietnam War, there was a federal surtax imposed on telephone -service; Hoffman and his cohorts could, and did, argue that in systematically -stealing phone service they were engaging in civil disobedience: -virtuously denying tax funds to an illegal and immoral war. - -But this thin veil of decency was soon dropped entirely. -Ripping-off the System found its own justification in deep alienation -and a basic outlaw contempt for conventional bourgeois values. -Ingenious, vaguely politicized varieties of rip-off, -which might be described as "anarchy by convenience," -became very popular in Yippie circles, and because rip-off -was so useful, it was to survive the Yippie movement itself. - -In the early 1970s, it required fairly limited expertise -and ingenuity to cheat payphones, to divert "free" -electricity and gas service, or to rob vending machines -and parking meters for handy pocket change. It also required -a conspiracy to spread this knowledge, and the gall -and nerve actually to commit petty theft, but the Yippies -had these qualifications in plenty. In June 1971, Abbie -Hoffman and a telephone enthusiast sarcastically known -as "Al Bell" began publishing a newsletter called Youth -International Party Line. This newsletter was dedicated -to collating and spreading Yippie rip-off techniques, -especially of phones, to the joy of the freewheeling -underground and the insensate rage of all straight people. -As a political tactic, phone-service theft ensured -that Yippie advocates would always have ready access -to the long-distance telephone as a medium, despite -the Yippies' chronic lack of organization, discipline, -money, or even a steady home address. - -PARTY LINE was run out of Greenwich Village for a couple of years, -then "Al Bell" more or less defected from the faltering ranks of Yippiedom, -changing the newsletter's name to TAP or Technical Assistance Program. -After the Vietnam War ended, the steam began leaking rapidly out of American -radical dissent. But by this time, "Bell" and his dozen or so -core contributors had the bit between their teeth, -and had begun to derive tremendous gut-level satisfaction -from the sensation of pure TECHNICAL POWER. - -TAP articles, once highly politicized, became pitilessly jargonized -and technical, in homage or parody to the Bell System's own technical -documents, which TAP studied closely, gutted, and reproduced without -permission. The TAP elite revelled in gloating possession -of the specialized knowledge necessary to beat the system. - -"Al Bell" dropped out of the game by the late 70s, -and "Tom Edison" took over; TAP readers (some 1400 of -them, all told) now began to show more interest in telex -switches and the growing phenomenon of computer systems. - -In 1983, "Tom Edison" had his computer stolen and his house -set on fire by an arsonist. This was an eventually mortal blow -to TAP (though the legendary name was to be resurrected -in 1990 by a young Kentuckian computer-outlaw named "Predat0r.") - -# - -Ever since telephones began to make money, there have been -people willing to rob and defraud phone companies. -The legions of petty phone thieves vastly outnumber those -"phone phreaks" who "explore the system" for the sake -of the intellectual challenge. The New York metropolitan area -(long in the vanguard of American crime) claims over 150,000 -physical attacks on pay telephones every year! Studied carefully, -a modern payphone reveals itself as a little fortress, carefully -designed and redesigned over generations, to resist coin-slugs, -zaps of electricity, chunks of coin-shaped ice, prybars, magnets, -lockpicks, blasting caps. Public pay- phones must survive in a world -of unfriendly, greedy people, and a modern payphone is as exquisitely -evolved as a cactus. -Because the phone network pre-dates the computer network, -the scofflaws known as "phone phreaks" pre-date the scofflaws -known as "computer hackers." In practice, today, the line -between "phreaking" and "hacking" is very blurred, -just as the distinction between telephones and computers -has blurred. The phone system has been digitized, -and computers have learned to "talk" over phone-lines. -What's worse--and this was the point of the Mr. Jenkins -of the Secret Service--some hackers have learned to steal, -and some thieves have learned to hack. - -Despite the blurring, one can still draw a few useful -behavioral distinctions between "phreaks" and "hackers." -Hackers are intensely interested in the "system" per se, -and enjoy relating to machines. "Phreaks" are more -social, manipulating the system in a rough-and-ready -fashion in order to get through to other human beings, -fast, cheap and under the table. - -Phone phreaks love nothing so much as "bridges," -illegal conference calls of ten or twelve chatting -conspirators, seaboard to seaboard, lasting for many hours ---and running, of course, on somebody else's tab, -preferably a large corporation's. - -As phone-phreak conferences wear on, people drop out -(or simply leave the phone off the hook, while they -sashay off to work or school or babysitting), -and new people are phoned up and invited to join in, -from some other continent, if possible. Technical trivia, -boasts, brags, lies, head-trip deceptions, weird rumors, -and cruel gossip are all freely exchanged. - -The lowest rung of phone-phreaking is the theft of telephone access codes. -Charging a phone call to somebody else's stolen number is, of course, -a pig-easy way of stealing phone service, requiring practically no -technical expertise. This practice has been very widespread, -especially among lonely people without much money who are far from home. -Code theft has flourished especially in college dorms, military bases, -and, notoriously, among roadies for rock bands. Of late, code theft -has spread very rapidly among Third Worlders in the US, who pile up -enormous unpaid long-distance bills to the Caribbean, South America, -and Pakistan. - -The simplest way to steal phone-codes is simply to look over -a victim's shoulder as he punches-in his own code-number -on a public payphone. This technique is known as "shoulder-surfing," -and is especially common in airports, bus terminals, and train stations. -The code is then sold by the thief for a few dollars. The buyer abusing -the code has no computer expertise, but calls his Mom in New York, -Kingston or Caracas and runs up a huge bill with impunity. The losses -from this primitive phreaking activity are far, far greater than the -monetary losses caused by computer-intruding hackers. - -In the mid-to-late 1980s, until the introduction of sterner telco -security measures, COMPUTERIZED code theft worked like a charm, -and was virtually omnipresent throughout the digital underground, -among phreaks and hackers alike. This was accomplished through -programming one's computer to try random code numbers over the telephone -until one of them worked. Simple programs to do this were widely available -in the underground; a computer running all night was likely to come up with -a dozen or so useful hits. This could be repeated week after week until -one had a large library of stolen codes. - -Nowadays, the computerized dialling of hundreds of numbers -can be detected within hours and swiftly traced. -If a stolen code is repeatedly abused, this too can -be detected within a few hours. But for years in the 1980s, -the publication of stolen codes was a kind of elementary etiquette -for fledgling hackers. The simplest way to establish your bona-fides -as a raider was to steal a code through repeated random dialling -and offer it to the "community" for use. Codes could be both stolen, -and used, simply and easily from the safety of one's own bedroom, -with very little fear of detection or punishment. - -Before computers and their phone-line modems entered American homes -in gigantic numbers, phone phreaks had their own special telecommunications -hardware gadget, the famous "blue box." This fraud device (now rendered -increasingly useless by the digital evolution of the phone system) could -trick switching systems into granting free access to long-distance lines. -It did this by mimicking the system's own signal, a tone of 2600 hertz. - -Steven Jobs and Steve Wozniak, the founders of Apple Computer, Inc., -once dabbled in selling blue-boxes in college dorms in California. -For many, in the early days of phreaking, blue-boxing was scarcely -perceived as "theft," but rather as a fun (if sneaky) way to use -excess phone capacity harmlessly. After all, the long-distance -lines were JUST SITTING THERE. . . . Whom did it hurt, really? -If you're not DAMAGING the system, and you're not USING UP ANY -TANGIBLE RESOURCE, and if nobody FINDS OUT what you did, -then what real harm have you done? What exactly HAVE you "stolen," -anyway? If a tree falls in the forest and nobody hears it, -how much is the noise worth? Even now this remains a rather -dicey question. - -Blue-boxing was no joke to the phone companies, however. -Indeed, when Ramparts magazine, a radical publication in California, -printed the wiring schematics necessary to create a mute box in June 1972, -the magazine was seized by police and Pacific Bell phone-company officials. -The mute box, a blue-box variant, allowed its user to receive long-distance -calls free of charge to the caller. This device was closely described in a -Ramparts article wryly titled "Regulating the Phone Company In Your Home." -Publication of this article was held to be in violation of Californian -State Penal Code section 502.7, which outlaws ownership of wire-fraud -devices and the selling of "plans or instructions for any instrument, -apparatus, or device intended to avoid telephone toll charges." - -Issues of Ramparts were recalled or seized on the newsstands, -and the resultant loss of income helped put the magazine out of business. -This was an ominous precedent for free-expression issues, but the telco's -crushing of a radical-fringe magazine passed without serious challenge -at the time. Even in the freewheeling California 1970s, it was widely felt -that there was something sacrosanct about what the phone company knew; -that the telco had a legal and moral right to protect itself by shutting -off the flow of such illicit information. Most telco information was so -"specialized" that it would scarcely be understood by any honest member -of the public. If not published, it would not be missed. To print such -material did not seem part of the legitimate role of a free press. - -In 1990 there would be a similar telco-inspired attack -on the electronic phreak/hacking "magazine" Phrack. -The Phrack legal case became a central issue in the -Hacker Crackdown, and gave rise to great controversy. -Phrack would also be shut down, for a time, at least, -but this time both the telcos and their law-enforcement -allies would pay a much larger price for their actions. -The Phrack case will be examined in detail, later. - -Phone-phreaking as a social practice is still very -much alive at this moment. Today, phone-phreaking -is thriving much more vigorously than the better-known -and worse-feared practice of "computer hacking." -New forms of phreaking are spreading rapidly, following -new vulnerabilities in sophisticated phone services. - -Cellular phones are especially vulnerable; their chips -can be re-programmed to present a false caller ID -and avoid billing. Doing so also avoids police tapping, -making cellular-phone abuse a favorite among drug-dealers. -"Call-sell operations" using pirate cellular phones can, -and have, been run right out of the backs of cars, which move -from "cell" to "cell" in the local phone system, retailing -stolen long-distance service, like some kind of demented -electronic version of the neighborhood ice-cream truck. - -Private branch-exchange phone systems in large corporations -can be penetrated; phreaks dial-up a local company, enter its -internal phone-system, hack it, then use the company's own -PBX system to dial back out over the public network, -causing the company to be stuck with the resulting -long-distance bill. This technique is known as "diverting." -"Diverting" can be very costly, especially because phreaks -tend to travel in packs and never stop talking. -Perhaps the worst by-product of this "PBX fraud" -is that victim companies and telcos have sued one another -over the financial responsibility for the stolen calls, -thus enriching not only shabby phreaks but well-paid lawyers. - -"Voice-mail systems" can also be abused; phreaks -can seize their own sections of these sophisticated -electronic answering machines, and use them for trading -codes or knowledge of illegal techniques. Voice-mail -abuse does not hurt the company directly, but finding -supposedly empty slots in your company's answering -machine all crammed with phreaks eagerly chattering -and hey-duding one another in impenetrable jargon can -cause sensations of almost mystical repulsion and dread. - -Worse yet, phreaks have sometimes been known to react -truculently to attempts to "clean up" the voice-mail system. -Rather than humbly acquiescing to being thrown out of their playground, -they may very well call up the company officials at work (or at home) -and loudly demand free voice-mail addresses of their very own. -Such bullying is taken very seriously by spooked victims. - -Acts of phreak revenge against straight people are rare, -but voice-mail systems are especially tempting and vulnerable, -and an infestation of angry phreaks in one's voice-mail system is no joke. -They can erase legitimate messages; or spy on private messages; -or harass users with recorded taunts and obscenities. -They've even been known to seize control of voice-mail security, -and lock out legitimate users, or even shut down the system entirely. - -Cellular phone-calls, cordless phones, and ship-to-shore -telephony can all be monitored by various forms of radio; -this kind of "passive monitoring" is spreading explosively today. -Technically eavesdropping on other people's cordless and cellular -phone-calls is the fastest-growing area in phreaking today. -This practice strongly appeals to the lust for power and conveys -gratifying sensations of technical superiority over the eavesdropping -victim. Monitoring is rife with all manner of tempting evil mischief. -Simple prurient snooping is by far the most common activity. -But credit-card numbers unwarily spoken over the phone can be recorded, -stolen and used. And tapping people's phone-calls (whether through -active telephone taps or passive radio monitors) does lend itself -conveniently to activities like blackmail, industrial espionage, -and political dirty tricks. - -It should be repeated that telecommunications fraud, -the theft of phone service, causes vastly greater monetary -losses than the practice of entering into computers by stealth. -Hackers are mostly young suburban American white males, -and exist in their hundreds--but "phreaks" come from both sexes -and from many nationalities, ages and ethnic backgrounds, -and are flourishing in the thousands. - -# - -The term "hacker" has had an unfortunate history. -This book, The Hacker Crackdown, has little to say about -"hacking" in its finer, original sense. The term can signify -the free-wheeling intellectual exploration of the highest -and deepest potential of computer systems. Hacking can -describe the determination to make access to computers -and information as free and open as possible. Hacking -can involve the heartfelt conviction that beauty can -be found in computers, that the fine aesthetic in a perfect -program can liberate the mind and spirit. This is "hacking" -as it was defined in Steven Levy's much-praised history -of the pioneer computer milieu, Hackers, published in 1984. - -Hackers of all kinds are absolutely soaked through with heroic -anti-bureaucratic sentiment. Hackers long for recognition -as a praiseworthy cultural archetype, the postmodern electronic -equivalent of the cowboy and mountain man. Whether they deserve -such a reputation is something for history to decide. But many hackers-- -including those outlaw hackers who are computer intruders, and whose -activities are defined as criminal--actually attempt to LIVE UP TO -this techno-cowboy reputation. And given that electronics and -telecommunications are still largely unexplored territories, -there is simply NO TELLING what hackers might uncover. - -For some people, this freedom is the very breath of oxygen, -the inventive spontaneity that makes life worth living -and that flings open doors to marvellous possibility and -individual empowerment. But for many people ---and increasingly so--the hacker is an ominous figure, -a smart-aleck sociopath ready to burst out of his basement -wilderness and savage other people's lives for his own -anarchical convenience. - -Any form of power without responsibility, without direct -and formal checks and balances, is frightening to people-- -and reasonably so. It should be frankly admitted that -hackers ARE frightening, and that the basis of this fear -is not irrational. - -Fear of hackers goes well beyond the fear of merely criminal activity. - -Subversion and manipulation of the phone system -is an act with disturbing political overtones. -In America, computers and telephones are potent symbols -of organized authority and the technocratic business elite. - -But there is an element in American culture that -has always strongly rebelled against these symbols; -rebelled against all large industrial computers -and all phone companies. A certain anarchical tinge deep -in the American soul delights in causing confusion and pain -to all bureaucracies, including technological ones. - -There is sometimes malice and vandalism in this attitude, -but it is a deep and cherished part of the American national character. -The outlaw, the rebel, the rugged individual, the pioneer, -the sturdy Jeffersonian yeoman, the private citizen resisting -interference in his pursuit of happiness--these are figures that all -Americans recognize, and that many will strongly applaud and defend. - -Many scrupulously law-abiding citizens today do cutting-edge work -with electronics--work that has already had tremendous social influence -and will have much more in years to come. In all truth, these talented, -hardworking, law-abiding, mature, adult people are far more disturbing -to the peace and order of the current status quo than any scofflaw group -of romantic teenage punk kids. These law-abiding hackers have the power, -ability, and willingness to influence other people's lives quite unpredictably. -They have means, motive, and opportunity to meddle drastically with the -American social order. When corralled into governments, universities, -or large multinational companies, and forced to follow rulebooks -and wear suits and ties, they at least have some conventional halters -on their freedom of action. But when loosed alone, or in small groups, -and fired by imagination and the entrepreneurial spirit, they can move -mountains--causing landslides that will likely crash directly into your -office and living room. - -These people, as a class, instinctively recognize that a public, -politicized attack on hackers will eventually spread to them-- -that the term "hacker," once demonized, might be used to knock -their hands off the levers of power and choke them out of existence. -There are hackers today who fiercely and publicly resist any besmirching -of the noble title of hacker. Naturally and understandably, they deeply -resent the attack on their values implicit in using the word "hacker" -as a synonym for computer-criminal. - -This book, sadly but in my opinion unavoidably, rather adds -to the degradation of the term. It concerns itself mostly with "hacking" -in its commonest latter-day definition, i.e., intruding into computer -systems by stealth and without permission. The term "hacking" is used -routinely today by almost all law enforcement officials with any -professional interest in computer fraud and abuse. American police -describe almost any crime committed with, by, through, or against -a computer as hacking. - -Most importantly, "hacker" is what computer-intruders -choose to call THEMSELVES. Nobody who "hacks" into systems -willingly describes himself (rarely, herself) as a "computer intruder," -"computer trespasser," "cracker," "wormer," "darkside hacker" -or "high tech street gangster." Several other demeaning terms -have been invented in the hope that the press and public -will leave the original sense of the word alone. But few people -actually use these terms. (I exempt the term "cyberpunk," -which a few hackers and law enforcement people actually do use. -The term "cyberpunk" is drawn from literary criticism and has -some odd and unlikely resonances, but, like hacker, -cyberpunk too has become a criminal pejorative today.) - -In any case, breaking into computer systems was hardly alien -to the original hacker tradition. The first tottering systems -of the 1960s required fairly extensive internal surgery merely -to function day-by-day. Their users "invaded" the deepest, -most arcane recesses of their operating software almost -as a matter of routine. "Computer security" in these early, -primitive systems was at best an afterthought. What security -there was, was entirely physical, for it was assumed that -anyone allowed near this expensive, arcane hardware would be -a fully qualified professional expert. - -In a campus environment, though, this meant that grad students, -teaching assistants, undergraduates, and eventually, -all manner of dropouts and hangers-on ended up accessing -and often running the works. - -Universities, even modern universities, are not in -the business of maintaining security over information. -On the contrary, universities, as institutions, pre-date -the "information economy" by many centuries and are not- -for-profit cultural entities, whose reason for existence -(purportedly) is to discover truth, codify it through -techniques of scholarship, and then teach it. Universities -are meant to PASS THE TORCH OF CIVILIZATION, not just -download data into student skulls, and the values of the -academic community are strongly at odds with those of all -would-be information empires. Teachers at all levels, from -kindergarten up, have proven to be shameless and persistent -software and data pirates. Universities do not merely -"leak information" but vigorously broadcast free thought. - -This clash of values has been fraught with controversy. -Many hackers of the 1960s remember their professional -apprenticeship as a long guerilla war against the uptight -mainframe-computer "information priesthood." These computer-hungry -youngsters had to struggle hard for access to computing power, -and many of them were not above certain, er, shortcuts. -But, over the years, this practice freed computing -from the sterile reserve of lab-coated technocrats and -was largely responsible for the explosive growth of computing -in general society--especially PERSONAL computing. - -Access to technical power acted like catnip on certain -of these youngsters. Most of the basic techniques of -computer intrusion: password cracking, trapdoors, backdoors, -trojan horses--were invented in college environments in the 1960s, -in the early days of network computing. Some off-the-cuff -experience at computer intrusion was to be in the informal -resume of most "hackers" and many future industry giants. -Outside of the tiny cult of computer enthusiasts, few people -thought much about the implications of "breaking into" -computers. This sort of activity had not yet been publicized, -much less criminalized. - -In the 1960s, definitions of "property" and "privacy" -had not yet been extended to cyberspace. Computers -were not yet indispensable to society. There were no vast -databanks of vulnerable, proprietary information stored -in computers, which might be accessed, copied without -permission, erased, altered, or sabotaged. The stakes -were low in the early days--but they grew every year, -exponentially, as computers themselves grew. - -By the 1990s, commercial and political pressures -had become overwhelming, and they broke the social -boundaries of the hacking subculture. Hacking -had become too important to be left to the hackers. -Society was now forced to tackle the intangible nature -of cyberspace-as-property, cyberspace as privately-owned -unreal-estate. In the new, severe, responsible, high-stakes -context of the "Information Society" of the 1990s, -"hacking" was called into question. - -What did it mean to break into a computer without -permission and use its computational power, or look -around inside its files without hurting anything? -What were computer-intruding hackers, anyway--how should -society, and the law, best define their actions? -Were they just BROWSERS, harmless intellectual explorers? -Were they VOYEURS, snoops, invaders of privacy? Should -they be sternly treated as potential AGENTS OF ESPIONAGE, -or perhaps as INDUSTRIAL SPIES? Or were they best -defined as TRESPASSERS, a very common teenage -misdemeanor? Was hacking THEFT OF SERVICE? -(After all, intruders were getting someone else's -computer to carry out their orders, without permission -and without paying). Was hacking FRAUD? Maybe it was -best described as IMPERSONATION. The commonest mode -of computer intrusion was (and is) to swipe or snoop -somebody else's password, and then enter the computer -in the guise of another person--who is commonly stuck -with the blame and the bills. - -Perhaps a medical metaphor was better--hackers should -be defined as "sick," as COMPUTER ADDICTS unable -to control their irresponsible, compulsive behavior. - -But these weighty assessments meant little to the -people who were actually being judged. From inside -the underground world of hacking itself, all these -perceptions seem quaint, wrongheaded, stupid, or meaningless. -The most important self-perception of underground hackers-- -from the 1960s, right through to the present day--is that -they are an ELITE. The day-to-day struggle in the underground -is not over sociological definitions--who cares?--but for power, -knowledge, and status among one's peers. - -When you are a hacker, it is your own inner conviction -of your elite status that enables you to break, or let -us say "transcend," the rules. It is not that ALL rules -go by the board. The rules habitually broken by hackers -are UNIMPORTANT rules--the rules of dopey greedhead telco -bureaucrats and pig-ignorant government pests. - -Hackers have their OWN rules, which separate behavior -which is cool and elite, from behavior which is rodentlike, -stupid and losing. These "rules," however, are mostly unwritten -and enforced by peer pressure and tribal feeling. Like all rules -that depend on the unspoken conviction that everybody else -is a good old boy, these rules are ripe for abuse. The mechanisms -of hacker peer- pressure, "teletrials" and ostracism, are rarely used -and rarely work. Back-stabbing slander, threats, and electronic -harassment are also freely employed in down-and-dirty intrahacker feuds, -but this rarely forces a rival out of the scene entirely. The only real -solution for the problem of an utterly losing, treacherous and rodentlike -hacker is to TURN HIM IN TO THE POLICE. Unlike the Mafia or Medellin Cartel, -the hacker elite cannot simply execute the bigmouths, creeps and troublemakers -among their ranks, so they turn one another in with astonishing frequency. - -There is no tradition of silence or OMERTA in the hacker underworld. -Hackers can be shy, even reclusive, but when they do talk, hackers -tend to brag, boast and strut. Almost everything hackers do is INVISIBLE; -if they don't brag, boast, and strut about it, then NOBODY WILL EVER KNOW. -If you don't have something to brag, boast, and strut about, then nobody -in the underground will recognize you and favor you with vital cooperation -and respect. - -The way to win a solid reputation in the underground -is by telling other hackers things that could only -have been learned by exceptional cunning and stealth. -Forbidden knowledge, therefore, is the basic currency -of the digital underground, like seashells among -Trobriand Islanders. Hackers hoard this knowledge, -and dwell upon it obsessively, and refine it, -and bargain with it, and talk and talk about it. - -Many hackers even suffer from a strange obsession to TEACH-- -to spread the ethos and the knowledge of the digital underground. -They'll do this even when it gains them no particular advantage -and presents a grave personal risk. - -And when that risk catches up with them, they will go right on teaching -and preaching--to a new audience this time, their interrogators from law -enforcement. Almost every hacker arrested tells everything he knows-- -all about his friends, his mentors, his disciples--legends, threats, -horror stories, dire rumors, gossip, hallucinations. This is, of course, -convenient for law enforcement--except when law enforcement begins -to believe hacker legendry. - -Phone phreaks are unique among criminals in their willingness -to call up law enforcement officials--in the office, at their homes-- -and give them an extended piece of their mind. It is hard not to -interpret this as BEGGING FOR ARREST, and in fact it is an act -of incredible foolhardiness. Police are naturally nettled -by these acts of chutzpah and will go well out of their way -to bust these flaunting idiots. But it can also be interpreted -as a product of a world-view so elitist, so closed and hermetic, -that electronic police are simply not perceived as "police," -but rather as ENEMY PHONE PHREAKS who should be scolded -into behaving "decently." - -Hackers at their most grandiloquent perceive themselves -as the elite pioneers of a new electronic world. -Attempts to make them obey the democratically -established laws of contemporary American society are -seen as repression and persecution. After all, they argue, -if Alexander Graham Bell had gone along with the rules -of the Western Union telegraph company, there would have -been no telephones. If Jobs and Wozniak had believed -that IBM was the be-all and end-all, there would have -been no personal computers. If Benjamin Franklin and -Thomas Jefferson had tried to "work within the system" -there would have been no United States. - -Not only do hackers privately believe this as an article of faith, -but they have been known to write ardent manifestos about it. -Here are some revealing excerpts from an especially vivid hacker manifesto: -"The Techno-Revolution" by "Dr. Crash," which appeared in electronic -form in Phrack Volume 1, Issue 6, Phile 3. - - -"To fully explain the true motives behind hacking, -we must first take a quick look into the past. In the 1960s, -a group of MIT students built the first modern computer system. -This wild, rebellious group of young men were the first to bear -the name `hackers.' The systems that they developed were intended -to be used to solve world problems and to benefit all of mankind. -"As we can see, this has not been the case. The computer system -has been solely in the hands of big businesses and the government. -The wonderful device meant to enrich life has become a weapon which -dehumanizes people. To the government and large businesses, -people are no more than disk space, and the government doesn't -use computers to arrange aid for the poor, but to control nuclear -death weapons. The average American can only have access -to a small microcomputer which is worth only a fraction -of what they pay for it. The businesses keep the -true state-of-the-art equipment away from the people -behind a steel wall of incredibly high prices and bureaucracy. -It is because of this state of affairs that hacking was born. (. . .) -"Of course, the government doesn't want the monopoly of technology broken, -so they have outlawed hacking and arrest anyone who is caught. (. . .) -The phone company is another example of technology abused and kept -from people with high prices. (. . .) "Hackers often find that their -existing equipment, due to the monopoly tactics of computer companies, -is inefficient for their purposes. Due to the exorbitantly high prices, -it is impossible to legally purchase the necessary equipment. -This need has given still another segment of the fight: Credit Carding. -Carding is a way of obtaining the necessary goods without paying for them. -It is again due to the companies' stupidity that Carding is so easy, -and shows that the world's businesses are in the hands of those -with considerably less technical know-how than we, the hackers. (. . .) -"Hacking must continue. We must train newcomers to the art of hacking. -(. . . .) And whatever you do, continue the fight. Whether you know it -or not, if you are a hacker, you are a revolutionary. Don't worry, -you're on the right side." - -The defense of "carding" is rare. Most hackers regard credit-card -theft as "poison" to the underground, a sleazy and immoral effort that, -worse yet, is hard to get away with. Nevertheless, manifestos advocating -credit-card theft, the deliberate crashing of computer systems, -and even acts of violent physical destruction such as vandalism -and arson do exist in the underground. These boasts and threats -are taken quite seriously by the police. And not every hacker -is an abstract, Platonic computer-nerd. Some few are quite experienced -at picking locks, robbing phone-trucks, and breaking and entering buildings. - -Hackers vary in their degree of hatred for authority -and the violence of their rhetoric. But, at a bottom line, -they are scofflaws. They don't regard the current rules -of electronic behavior as respectable efforts to preserve -law and order and protect public safety. They regard these -laws as immoral efforts by soulless corporations to protect -their profit margins and to crush dissidents. "Stupid" people, -including police, businessmen, politicians, and journalists, -simply have no right to judge the actions of those possessed of genius, -techno-revolutionary intentions, and technical expertise. - -# - -Hackers are generally teenagers and college kids not -engaged in earning a living. They often come from fairly -well-to-do middle-class backgrounds, and are markedly -anti-materialistic (except, that is, when it comes to -computer equipment). Anyone motivated by greed for -mere money (as opposed to the greed for power, -knowledge and status) is swiftly written-off as a narrow- -minded breadhead whose interests can only be corrupt -and contemptible. Having grown up in the 1970s and -1980s, the young Bohemians of the digital underground -regard straight society as awash in plutocratic corruption, -where everyone from the President down is for sale and -whoever has the gold makes the rules. - -Interestingly, there's a funhouse-mirror image of this attitude -on the other side of the conflict. The police are also -one of the most markedly anti-materialistic groups -in American society, motivated not by mere money -but by ideals of service, justice, esprit-de-corps, -and, of course, their own brand of specialized knowledge -and power. Remarkably, the propaganda war between cops -and hackers has always involved angry allegations -that the other side is trying to make a sleazy buck. -Hackers consistently sneer that anti-phreak prosecutors -are angling for cushy jobs as telco lawyers and that -computer-crime police are aiming to cash in later -as well-paid computer-security consultants in the private sector. - -For their part, police publicly conflate all -hacking crimes with robbing payphones with crowbars. -Allegations of "monetary losses" from computer intrusion -are notoriously inflated. The act of illicitly copying -a document from a computer is morally equated with -directly robbing a company of, say, half a million dollars. -The teenage computer intruder in possession of this "proprietary" -document has certainly not sold it for such a sum, would likely -have little idea how to sell it at all, and quite probably -doesn't even understand what he has. He has not made a cent -in profit from his felony but is still morally equated with -a thief who has robbed the church poorbox and lit out for Brazil. - -Police want to believe that all hackers are thieves. -It is a tortuous and almost unbearable act for the American -justice system to put people in jail because they want -to learn things which are forbidden for them to know. -In an American context, almost any pretext for punishment -is better than jailing people to protect certain restricted -kinds of information. Nevertheless, POLICING INFORMATION -is part and parcel of the struggle against hackers. - -This dilemma is well exemplified by the remarkable -activities of "Emmanuel Goldstein," editor and publisher -of a print magazine known as 2600: The Hacker Quarterly. -Goldstein was an English major at Long Island's State University -of New York in the '70s, when he became involved with the local -college radio station. His growing interest in electronics -caused him to drift into Yippie TAP circles and thus into -the digital underground, where he became a self-described -techno-rat. His magazine publishes techniques of computer -intrusion and telephone "exploration" as well as gloating -exposes of telco misdeeds and governmental failings. - -Goldstein lives quietly and very privately in a large, -crumbling Victorian mansion in Setauket, New York. -The seaside house is decorated with telco decals, chunks of -driftwood, and the basic bric-a-brac of a hippie crash-pad. -He is unmarried, mildly unkempt, and survives mostly -on TV dinners and turkey-stuffing eaten straight out -of the bag. Goldstein is a man of considerable charm -and fluency, with a brief, disarming smile and the kind -of pitiless, stubborn, thoroughly recidivist integrity -that America's electronic police find genuinely alarming. - -Goldstein took his nom-de-plume, or "handle," from -a character in Orwell's 1984, which may be taken, -correctly, as a symptom of the gravity of his sociopolitical -worldview. He is not himself a practicing computer -intruder, though he vigorously abets these actions, -especially when they are pursued against large -corporations or governmental agencies. Nor is he a thief, -for he loudly scorns mere theft of phone service, in favor -of "exploring and manipulating the system." He is probably -best described and understood as a DISSIDENT. - -Weirdly, Goldstein is living in modern America -under conditions very similar to those of former -East European intellectual dissidents. In other words, -he flagrantly espouses a value-system that is deeply -and irrevocably opposed to the system of those in power -and the police. The values in 2600 are generally expressed -in terms that are ironic, sarcastic, paradoxical, or just -downright confused. But there's no mistaking their -radically anti-authoritarian tenor. 2600 holds that -technical power and specialized knowledge, of any kind -obtainable, belong by right in the hands of those individuals -brave and bold enough to discover them--by whatever means necessary. -Devices, laws, or systems that forbid access, and the free -spread of knowledge, are provocations that any free -and self-respecting hacker should relentlessly attack. -The "privacy" of governments, corporations and other soulless -technocratic organizations should never be protected -at the expense of the liberty and free initiative -of the individual techno-rat. - -However, in our contemporary workaday world, both governments -and corporations are very anxious indeed to police information -which is secret, proprietary, restricted, confidential, -copyrighted, patented, hazardous, illegal, unethical, -embarrassing, or otherwise sensitive. This makes Goldstein -persona non grata, and his philosophy a threat. - -Very little about the conditions of Goldstein's daily -life would astonish, say, Vaclav Havel. (We may note -in passing that President Havel once had his word-processor -confiscated by the Czechoslovak police.) Goldstein lives -by SAMIZDAT, acting semi-openly as a data-center -for the underground, while challenging the powers-that-be -to abide by their own stated rules: freedom of speech -and the First Amendment. - -Goldstein thoroughly looks and acts the part of techno-rat, -with shoulder-length ringlets and a piratical black -fisherman's-cap set at a rakish angle. He often shows up -like Banquo's ghost at meetings of computer professionals, -where he listens quietly, half-smiling and taking thorough notes. - -Computer professionals generally meet publicly, -and find it very difficult to rid themselves of Goldstein -and his ilk without extralegal and unconstitutional actions. -Sympathizers, many of them quite respectable people -with responsible jobs, admire Goldstein's attitude and -surreptitiously pass him information. An unknown but -presumably large proportion of Goldstein's 2,000-plus -readership are telco security personnel and police, -who are forced to subscribe to 2600 to stay abreast -of new developments in hacking. They thus find themselves -PAYING THIS GUY'S RENT while grinding their teeth in anguish, -a situation that would have delighted Abbie Hoffman -(one of Goldstein's few idols). - -Goldstein is probably the best-known public representative -of the hacker underground today, and certainly the best-hated. -Police regard him as a Fagin, a corrupter of youth, and speak -of him with untempered loathing. He is quite an accomplished gadfly. -After the Martin Luther King Day Crash of 1990, Goldstein, -for instance, adeptly rubbed salt into the wound in the pages of 2600. -"Yeah, it was fun for the phone phreaks as we watched the network crumble," -he admitted cheerfully. "But it was also an ominous sign of what's -to come. . . . Some AT&T people, aided by well-meaning but ignorant media, -were spreading the notion that many companies had the same software -and therefore could face the same problem someday. Wrong. This was -entirely an AT&T software deficiency. Of course, other companies could -face entirely DIFFERENT software problems. But then, so too could AT&T." - -After a technical discussion of the system's failings, -the Long Island techno-rat went on to offer thoughtful -criticism to the gigantic multinational's hundreds of -professionally qualified engineers. "What we don't know -is how a major force in communications like AT&T could -be so sloppy. What happened to backups? Sure, -computer systems go down all the time, but people -making phone calls are not the same as people logging -on to computers. We must make that distinction. It's not -acceptable for the phone system or any other essential -service to `go down.' If we continue to trust technology -without understanding it, we can look forward to many -variations on this theme. - -"AT&T owes it to its customers to be prepared to INSTANTLY -switch to another network if something strange and unpredictable -starts occurring. The news here isn't so much the failure -of a computer program, but the failure of AT&T's entire structure." - -The very idea of this. . . . this PERSON. . . . offering -"advice" about "AT&T's entire structure" is more than -some people can easily bear. How dare this near-criminal -dictate what is or isn't "acceptable" behavior from AT&T? -Especially when he's publishing, in the very same issue, -detailed schematic diagrams for creating various switching-network -signalling tones unavailable to the public. - -"See what happens when you drop a `silver box' tone or two -down your local exchange or through different long distance -service carriers," advises 2600 contributor "Mr. Upsetter" -in "How To Build a Signal Box." "If you experiment systematically -and keep good records, you will surely discover something interesting." - -This is, of course, the scientific method, generally regarded -as a praiseworthy activity and one of the flowers of modern civilization. -One can indeed learn a great deal with this sort of structured -intellectual activity. Telco employees regard this mode of "exploration" -as akin to flinging sticks of dynamite into their pond to see what lives -on the bottom. - -2600 has been published consistently since 1984. -It has also run a bulletin board computer system, -printed 2600 T-shirts, taken fax calls. . . . -The Spring 1991 issue has an interesting announcement on page 45: -"We just discovered an extra set of wires attached to our fax line -and heading up the pole. (They've since been clipped.) -Your faxes to us and to anyone else could be monitored." -In the worldview of 2600, the tiny band of techno-rat brothers -(rarely, sisters) are a beseiged vanguard of the truly free and honest. -The rest of the world is a maelstrom of corporate crime and high-level -governmental corruption, occasionally tempered with well-meaning -ignorance. To read a few issues in a row is to enter a nightmare -akin to Solzhenitsyn's, somewhat tempered by the fact that 2600 -is often extremely funny. - -Goldstein did not become a target of the Hacker Crackdown, -though he protested loudly, eloquently, and publicly about it, -and it added considerably to his fame. It was not that he is not -regarded as dangerous, because he is so regarded. Goldstein has had -brushes with the law in the past: in 1985, a 2600 bulletin board -computer was seized by the FBI, and some software on it was formally -declared "a burglary tool in the form of a computer program." -But Goldstein escaped direct repression in 1990, because his -magazine is printed on paper, and recognized as subject -to Constitutional freedom of the press protection. -As was seen in the Ramparts case, this is far from -an absolute guarantee. Still, as a practical matter, -shutting down 2600 by court-order would create so much -legal hassle that it is simply unfeasible, at least -for the present. Throughout 1990, both Goldstein -and his magazine were peevishly thriving. - -Instead, the Crackdown of 1990 would concern itself -with the computerized version of forbidden data. -The crackdown itself, first and foremost, was about -BULLETIN BOARD SYSTEMS. Bulletin Board Systems, most often -known by the ugly and un-pluralizable acronym "BBS," are -the life-blood of the digital underground. Boards were -also central to law enforcement's tactics and strategy -in the Hacker Crackdown. - -A "bulletin board system" can be formally defined as -a computer which serves as an information and message- -passing center for users dialing-up over the phone-lines -through the use of modems. A "modem," or modulator- -demodulator, is a device which translates the digital -impulses of computers into audible analog telephone -signals, and vice versa. Modems connect computers -to phones and thus to each other. - -Large-scale mainframe computers have been connected since the 1960s, -but PERSONAL computers, run by individuals out of their homes, -were first networked in the late 1970s. The "board" created -by Ward Christensen and Randy Suess in February 1978, -in Chicago, Illinois, is generally regarded as the first -personal-computer bulletin board system worthy of the name. - -Boards run on many different machines, employing many -different kinds of software. Early boards were crude and buggy, -and their managers, known as "system operators" or "sysops," -were hard-working technical experts who wrote their own software. -But like most everything else in the world of electronics, -boards became faster, cheaper, better-designed, and generally -far more sophisticated throughout the 1980s. They also moved -swiftly out of the hands of pioneers and into those of the -general public. By 1985 there were something in the -neighborhood of 4,000 boards in America. By 1990 it was -calculated, vaguely, that there were about 30,000 boards in -the US, with uncounted thousands overseas. - -Computer bulletin boards are unregulated enterprises. -Running a board is a rough-and-ready, catch-as-catch-can proposition. -Basically, anybody with a computer, modem, software and a phone-line -can start a board. With second-hand equipment and public-domain -free software, the price of a board might be quite small-- -less than it would take to publish a magazine or even a -decent pamphlet. Entrepreneurs eagerly sell bulletin-board -software, and will coach nontechnical amateur sysops in its use. - -Boards are not "presses." They are not magazines, -or libraries, or phones, or CB radios, or traditional cork -bulletin boards down at the local laundry, though they -have some passing resemblance to those earlier media. -Boards are a new medium--they may even be a LARGE NUMBER of new media. - -Consider these unique characteristics: boards are cheap, -yet they can have a national, even global reach. -Boards can be contacted from anywhere in the global -telephone network, at NO COST to the person running the board-- -the caller pays the phone bill, and if the caller is local, -the call is free. Boards do not involve an editorial elite -addressing a mass audience. The "sysop" of a board is not -an exclusive publisher or writer--he is managing an electronic salon, -where individuals can address the general public, play the part -of the general public, and also exchange private mail -with other individuals. And the "conversation" on boards, -though fluid, rapid, and highly interactive, is not spoken, -but written. It is also relatively anonymous, sometimes completely so. - -And because boards are cheap and ubiquitous, regulations -and licensing requirements would likely be practically unenforceable. -It would almost be easier to "regulate," "inspect," and "license" -the content of private mail--probably more so, since the mail system -is operated by the federal government. Boards are run by individuals, -independently, entirely at their own whim. - -For the sysop, the cost of operation is not the primary -limiting factor. Once the investment in a computer and -modem has been made, the only steady cost is the charge -for maintaining a phone line (or several phone lines). -The primary limits for sysops are time and energy. -Boards require upkeep. New users are generally "validated"-- -they must be issued individual passwords, and called at -home by voice-phone, so that their identity can be -verified. Obnoxious users, who exist in plenty, must be -chided or purged. Proliferating messages must be deleted -when they grow old, so that the capacity of the system -is not overwhelmed. And software programs (if such things -are kept on the board) must be examined for possible -computer viruses. If there is a financial charge to use -the board (increasingly common, especially in larger and -fancier systems) then accounts must be kept, and users -must be billed. And if the board crashes--a very common -occurrence--then repairs must be made. - -Boards can be distinguished by the amount of effort -spent in regulating them. First, we have the completely -open board, whose sysop is off chugging brews and -watching re-runs while his users generally degenerate -over time into peevish anarchy and eventual silence. -Second comes the supervised board, where the sysop -breaks in every once in a while to tidy up, calm brawls, -issue announcements, and rid the community of dolts -and troublemakers. Third is the heavily supervised -board, which sternly urges adult and responsible behavior -and swiftly edits any message considered offensive, -impertinent, illegal or irrelevant. And last comes -the completely edited "electronic publication," which -is presented to a silent audience which is not allowed -to respond directly in any way. - -Boards can also be grouped by their degree of anonymity. -There is the completely anonymous board, where everyone -uses pseudonyms--"handles"--and even the sysop is unaware -of the user's true identity. The sysop himself is likely -pseudonymous on a board of this type. Second, and rather -more common, is the board where the sysop knows (or thinks -he knows) the true names and addresses of all users, -but the users don't know one another's names and may not know his. -Third is the board where everyone has to use real names, -and roleplaying and pseudonymous posturing are forbidden. - -Boards can be grouped by their immediacy. "Chat-lines" -are boards linking several users together over several -different phone-lines simultaneously, so that people -exchange messages at the very moment that they type. -(Many large boards feature "chat" capabilities along -with other services.) Less immediate boards, -perhaps with a single phoneline, store messages serially, -one at a time. And some boards are only open for business -in daylight hours or on weekends, which greatly slows response. -A NETWORK of boards, such as "FidoNet," can carry electronic mail -from board to board, continent to continent, across huge distances-- -but at a relative snail's pace, so that a message can take several -days to reach its target audience and elicit a reply. - -Boards can be grouped by their degree of community. -Some boards emphasize the exchange of private, -person-to-person electronic mail. Others emphasize -public postings and may even purge people who "lurk," -merely reading posts but refusing to openly participate. -Some boards are intimate and neighborly. Others are frosty -and highly technical. Some are little more than storage -dumps for software, where users "download" and "upload" programs, -but interact among themselves little if at all. - -Boards can be grouped by their ease of access. Some boards -are entirely public. Others are private and restricted only -to personal friends of the sysop. Some boards divide users by status. -On these boards, some users, especially beginners, strangers or children, -will be restricted to general topics, and perhaps forbidden to post. -Favored users, though, are granted the ability to post as they please, -and to stay "on-line" as long as they like, even to the disadvantage -of other people trying to call in. High-status users can be given access -to hidden areas in the board, such as off-color topics, private discussions, -and/or valuable software. Favored users may even become "remote sysops" -with the power to take remote control of the board through their own -home computers. Quite often "remote sysops" end up doing all the work -and taking formal control of the enterprise, despite the fact that it's -physically located in someone else's house. Sometimes several "co-sysops" -share power. - -And boards can also be grouped by size. Massive, nationwide -commercial networks, such as CompuServe, Delphi, GEnie and Prodigy, -are run on mainframe computers and are generally not considered "boards," -though they share many of their characteristics, such as electronic mail, -discussion topics, libraries of software, and persistent and growing problems -with civil-liberties issues. Some private boards have as many as -thirty phone-lines and quite sophisticated hardware. And then -there are tiny boards. - -Boards vary in popularity. Some boards are huge and crowded, -where users must claw their way in against a constant busy-signal. -Others are huge and empty--there are few things sadder than a formerly -flourishing board where no one posts any longer, and the dead conversations -of vanished users lie about gathering digital dust. Some boards are tiny -and intimate, their telephone numbers intentionally kept confidential -so that only a small number can log on. - -And some boards are UNDERGROUND. - -Boards can be mysterious entities. The activities of -their users can be hard to differentiate from conspiracy. -Sometimes they ARE conspiracies. Boards have harbored, -or have been accused of harboring, all manner of fringe groups, -and have abetted, or been accused of abetting, every manner -of frowned-upon, sleazy, radical, and criminal activity. -There are Satanist boards. Nazi boards. Pornographic boards. -Pedophile boards. Drug- dealing boards. Anarchist boards. -Communist boards. Gay and Lesbian boards (these exist in great profusion, -many of them quite lively with well-established histories). -Religious cult boards. Evangelical boards. Witchcraft -boards, hippie boards, punk boards, skateboarder boards. -Boards for UFO believers. There may well be boards for -serial killers, airline terrorists and professional assassins. -There is simply no way to tell. Boards spring up, flourish, -and disappear in large numbers, in most every corner of -the developed world. Even apparently innocuous public -boards can, and sometimes do, harbor secret areas known -only to a few. And even on the vast, public, commercial services, -private mail is very private--and quite possibly criminal. - -Boards cover most every topic imaginable and some -that are hard to imagine. They cover a vast spectrum -of social activity. However, all board users do have -something in common: their possession of computers -and phones. Naturally, computers and phones are -primary topics of conversation on almost every board. - -And hackers and phone phreaks, those utter devotees -of computers and phones, live by boards. They swarm by boards. -They are bred by boards. By the late 1980s, phone-phreak groups -and hacker groups, united by boards, had proliferated fantastically. - - -As evidence, here is a list of hacker groups compiled -by the editors of Phrack on August 8, 1988. - - -The Administration. -Advanced Telecommunications, Inc. -ALIAS. -American Tone Travelers. -Anarchy Inc. -Apple Mafia. -The Association. -Atlantic Pirates Guild. - -Bad Ass Mother Fuckers. -Bellcore. -Bell Shock Force. -Black Bag. - -Camorra. -C&M Productions. -Catholics Anonymous. -Chaos Computer Club. -Chief Executive Officers. -Circle Of Death. -Circle Of Deneb. -Club X. -Coalition of Hi-Tech -Pirates. -Coast-To-Coast. -Corrupt Computing. -Cult Of The -Dead Cow. -Custom Retaliations. - -Damage Inc. -D&B Communications. -The Danger Gang. -Dec Hunters. -Digital Gang. -DPAK. - -Eastern Alliance. -The Elite Hackers Guild. -Elite Phreakers and Hackers Club. -The Elite Society Of America. -EPG. -Executives Of Crime. -Extasyy Elite. - -Fargo 4A. -Farmers Of Doom. -The Federation. -Feds R Us. -First Class. -Five O. -Five Star. -Force Hackers. -The 414s. - -Hack-A-Trip. -Hackers Of America. -High Mountain Hackers. -High Society. -The Hitchhikers. - -IBM Syndicate. -The Ice Pirates. -Imperial Warlords. -Inner Circle. -Inner Circle II. -Insanity Inc. -International Computer Underground Bandits. - -Justice League of America. - -Kaos Inc. -Knights Of Shadow. -Knights Of The Round Table. - -League Of Adepts. -Legion Of Doom. -Legion Of Hackers. -Lords Of Chaos. -Lunatic Labs, Unlimited. - -Master Hackers. -MAD! -The Marauders. -MD/PhD. - -Metal Communications, Inc. -MetalliBashers, Inc. -MBI. - -Metro Communications. -Midwest Pirates Guild. - -NASA Elite. -The NATO Association. -Neon Knights. - -Nihilist Order. -Order Of The Rose. -OSS. - -Pacific Pirates Guild. -Phantom Access Associates. - -PHido PHreaks. -The Phirm. -Phlash. -PhoneLine Phantoms. -Phone Phreakers Of America. -Phortune 500. - -Phreak Hack Delinquents. -Phreak Hack Destroyers. - -Phreakers, Hackers, And Laundromat Employees Gang (PHALSE Gang). -Phreaks Against Geeks. -Phreaks Against Phreaks Against Geeks. -Phreaks and Hackers of America. -Phreaks Anonymous World Wide. -Project Genesis. -The Punk Mafia. - -The Racketeers. -Red Dawn Text Files. -Roscoe Gang. - - -SABRE. -Secret Circle of Pirates. -Secret Service. -707 Club. -Shadow Brotherhood. -Sharp Inc. -65C02 Elite. - -Spectral Force. -Star League. -Stowaways. -Strata-Crackers. - - -Team Hackers '86. -Team Hackers '87. - -TeleComputist Newsletter Staff. -Tribunal Of Knowledge. - -Triple Entente. -Turn Over And Die Syndrome (TOADS). - -300 Club. -1200 Club. -2300 Club. -2600 Club. -2601 Club. - -2AF. - -The United Soft WareZ Force. -United Technical Underground. - -Ware Brigade. -The Warelords. -WASP. - -Contemplating this list is an impressive, almost humbling business. -As a cultural artifact, the thing approaches poetry. - -Underground groups--subcultures--can be distinguished -from independent cultures by their habit of referring -constantly to the parent society. Undergrounds by their -nature constantly must maintain a membrane of differentiation. -Funny/distinctive clothes and hair, specialized jargon, specialized -ghettoized areas in cities, different hours of rising, working, -sleeping. . . . The digital underground, which specializes in information, -relies very heavily on language to distinguish itself. As can be seen -from this list, they make heavy use of parody and mockery. -It's revealing to see who they choose to mock. - -First, large corporations. We have the Phortune 500, -The Chief Executive Officers, Bellcore, IBM Syndicate, -SABRE (a computerized reservation service maintained -by airlines). The common use of "Inc." is telling-- -none of these groups are actual corporations, -but take clear delight in mimicking them. - -Second, governments and police. NASA Elite, NATO Association. -"Feds R Us" and "Secret Service" are fine bits of fleering boldness. -OSS--the Office of Strategic Services was the forerunner of the CIA. - -Third, criminals. Using stigmatizing pejoratives as a perverse -badge of honor is a time-honored tactic for subcultures: -punks, gangs, delinquents, mafias, pirates, bandits, racketeers. - -Specialized orthography, especially the use of "ph" for "f" -and "z" for the plural "s," are instant recognition symbols. -So is the use of the numeral "0" for the letter "O" ---computer-software orthography generally features a -slash through the zero, making the distinction obvious. - -Some terms are poetically descriptive of computer intrusion: -the Stowaways, the Hitchhikers, the PhoneLine Phantoms, Coast-to-Coast. -Others are simple bravado and vainglorious puffery. -(Note the insistent use of the terms "elite" and "master.") -Some terms are blasphemous, some obscene, others merely cryptic-- -anything to puzzle, offend, confuse, and keep the straights at bay. - -Many hacker groups further re-encrypt their names -by the use of acronyms: United Technical Underground -becomes UTU, Farmers of Doom become FoD, the United SoftWareZ -Force becomes, at its own insistence, "TuSwF," and woe to the -ignorant rodent who capitalizes the wrong letters. - -It should be further recognized that the members of these groups -are themselves pseudonymous. If you did, in fact, run across -the "PhoneLine Phantoms," you would find them to consist of -"Carrier Culprit," "The Executioner," "Black Majik," -"Egyptian Lover," "Solid State," and "Mr Icom." -"Carrier Culprit" will likely be referred to by his friends -as "CC," as in, "I got these dialups from CC of PLP." - -It's quite possible that this entire list refers to as -few as a thousand people. It is not a complete list -of underground groups--there has never been such a list, -and there never will be. Groups rise, flourish, decline, -share membership, maintain a cloud of wannabes and -casual hangers-on. People pass in and out, are ostracized, -get bored, are busted by police, or are cornered by telco -security and presented with huge bills. Many "underground -groups" are software pirates, "warez d00dz," who might break -copy protection and pirate programs, but likely wouldn't dare -to intrude on a computer-system. - -It is hard to estimate the true population of the digital -underground. There is constant turnover. Most hackers -start young, come and go, then drop out at age 22-- -the age of college graduation. And a large majority -of "hackers" access pirate boards, adopt a handle, -swipe software and perhaps abuse a phone-code or two, -while never actually joining the elite. - -Some professional informants, who make it their business -to retail knowledge of the underground to paymasters in private -corporate security, have estimated the hacker population -at as high as fifty thousand. This is likely highly inflated, -unless one counts every single teenage software pirate -and petty phone-booth thief. My best guess is about 5,000 people. -Of these, I would guess that as few as a hundred are truly "elite" ---active computer intruders, skilled enough to penetrate -sophisticated systems and truly to worry corporate security -and law enforcement. - -Another interesting speculation is whether this group -is growing or not. Young teenage hackers are often -convinced that hackers exist in vast swarms and will soon -dominate the cybernetic universe. Older and wiser -veterans, perhaps as wizened as 24 or 25 years old, -are convinced that the glory days are long gone, that the cops -have the underground's number now, and that kids these days -are dirt-stupid and just want to play Nintendo. - -My own assessment is that computer intrusion, as a non-profit act -of intellectual exploration and mastery, is in slow decline, -at least in the United States; but that electronic fraud, -especially telecommunication crime, is growing by leaps and bounds. - -One might find a useful parallel to the digital underground -in the drug underground. There was a time, now much-obscured -by historical revisionism, when Bohemians freely shared joints -at concerts, and hip, small-scale marijuana dealers might -turn people on just for the sake of enjoying a long stoned conversation -about the Doors and Allen Ginsberg. Now drugs are increasingly verboten, -except in a high-stakes, highly-criminal world of highly addictive drugs. -Over years of disenchantment and police harassment, a vaguely ideological, -free-wheeling drug underground has relinquished the business of drug-dealing -to a far more savage criminal hard-core. This is not a pleasant prospect -to contemplate, but the analogy is fairly compelling. - -What does an underground board look like? What distinguishes -it from a standard board? It isn't necessarily the conversation-- -hackers often talk about common board topics, such as hardware, software, -sex, science fiction, current events, politics, movies, personal gossip. -Underground boards can best be distinguished by their files, or "philes," -pre-composed texts which teach the techniques and ethos of the underground. -These are prized reservoirs of forbidden knowledge. Some are anonymous, -but most proudly bear the handle of the "hacker" who has created them, -and his group affiliation, if he has one. - -Here is a partial table-of-contents of philes from an underground board, -somewhere in the heart of middle America, circa 1991. The descriptions -are mostly self-explanatory. - - -BANKAMER.ZIP 5406 06-11-91 Hacking Bank America -CHHACK.ZIP 4481 06-11-91 Chilton Hacking -CITIBANK.ZIP 4118 06-11-91 Hacking Citibank -CREDIMTC.ZIP 3241 06-11-91 Hacking Mtc Credit Company -DIGEST.ZIP 5159 06-11-91 Hackers Digest -HACK.ZIP 14031 06-11-91 How To Hack -HACKBAS.ZIP 5073 06-11-91 Basics Of Hacking -HACKDICT.ZIP 42774 06-11-91 Hackers Dictionary -HACKER.ZIP 57938 06-11-91 Hacker Info -HACKERME.ZIP 3148 06-11-91 Hackers Manual -HACKHAND.ZIP 4814 06-11-91 Hackers Handbook -HACKTHES.ZIP 48290 06-11-91 Hackers Thesis -HACKVMS.ZIP 4696 06-11-91 Hacking Vms Systems -MCDON.ZIP 3830 06-11-91 Hacking Macdonalds (Home Of The Archs) -P500UNIX.ZIP 15525 06-11-91 Phortune 500 Guide To Unix -RADHACK.ZIP 8411 06-11-91 Radio Hacking -TAOTRASH.DOC 4096 12-25-89 Suggestions For Trashing -TECHHACK.ZIP 5063 06-11-91 Technical Hacking - - -The files above are do-it-yourself manuals about computer intrusion. -The above is only a small section of a much larger library of hacking -and phreaking techniques and history. We now move into a different -and perhaps surprising area. - -+------------+ - |Anarchy| -+------------+ - -ANARC.ZIP 3641 06-11-91 Anarchy Files -ANARCHST.ZIP 63703 06-11-91 Anarchist Book -ANARCHY.ZIP 2076 06-11-91 Anarchy At Home -ANARCHY3.ZIP 6982 06-11-91 Anarchy No 3 -ANARCTOY.ZIP 2361 06-11-91 Anarchy Toys -ANTIMODM.ZIP 2877 06-11-91 Anti-modem Weapons -ATOM.ZIP 4494 06-11-91 How To Make An Atom Bomb -BARBITUA.ZIP 3982 06-11-91 Barbiturate Formula -BLCKPWDR.ZIP 2810 06-11-91 Black Powder Formulas -BOMB.ZIP 3765 06-11-91 How To Make Bombs -BOOM.ZIP 2036 06-11-91 Things That Go Boom -CHLORINE.ZIP 1926 06-11-91 Chlorine Bomb -COOKBOOK.ZIP 1500 06-11-91 Anarchy Cook Book -DESTROY.ZIP 3947 06-11-91 Destroy Stuff -DUSTBOMB.ZIP 2576 06-11-91 Dust Bomb -ELECTERR.ZIP 3230 06-11-91 Electronic Terror -EXPLOS1.ZIP 2598 06-11-91 Explosives 1 -EXPLOSIV.ZIP 18051 06-11-91 More Explosives -EZSTEAL.ZIP 4521 06-11-91 Ez-stealing -FLAME.ZIP 2240 06-11-91 Flame Thrower -FLASHLT.ZIP 2533 06-11-91 Flashlight Bomb -FMBUG.ZIP 2906 06-11-91 How To Make An Fm Bug -OMEEXPL.ZIP 2139 06-11-91 Home Explosives -HOW2BRK.ZIP 3332 06-11-91 How To Break In -LETTER.ZIP 2990 06-11-91 Letter Bomb -LOCK.ZIP 2199 06-11-91 How To Pick Locks -MRSHIN.ZIP 3991 06-11-91 Briefcase Locks -NAPALM.ZIP 3563 06-11-91 Napalm At Home -NITRO.ZIP 3158 06-11-91 Fun With Nitro -PARAMIL.ZIP 2962 06-11-91 Paramilitary Info -PICKING.ZIP 3398 06-11-91 Picking Locks -PIPEBOMB.ZIP 2137 06-11-91 Pipe Bomb -POTASS.ZIP 3987 06-11-91 Formulas With Potassium -PRANK.TXT 11074 08-03-90 More Pranks To Pull On Idiots! -REVENGE.ZIP 4447 06-11-91 Revenge Tactics -ROCKET.ZIP 2590 06-11-91 Rockets For Fun -SMUGGLE.ZIP 3385 06-11-91 How To Smuggle - -HOLY COW! The damned thing is full of stuff about bombs! - -What are we to make of this? - -First, it should be acknowledged that spreading -knowledge about demolitions to teenagers is a highly and -deliberately antisocial act. It is not, however, illegal. - -Second, it should be recognized that most of these -philes were in fact WRITTEN by teenagers. Most adult -American males who can remember their teenage years -will recognize that the notion of building a flamethrower -in your garage is an incredibly neat-o idea. ACTUALLY, -building a flamethrower in your garage, however, is -fraught with discouraging difficulty. Stuffing gunpowder -into a booby-trapped flashlight, so as to blow the arm off -your high-school vice-principal, can be a thing of dark -beauty to contemplate. Actually committing assault by -explosives will earn you the sustained attention of the -federal Bureau of Alcohol, Tobacco and Firearms. - -Some people, however, will actually try these plans. -A determinedly murderous American teenager can probably -buy or steal a handgun far more easily than he can brew -fake "napalm" in the kitchen sink. Nevertheless, -if temptation is spread before people, a certain number -will succumb, and a small minority will actually attempt -these stunts. A large minority of that small minority -will either fail or, quite likely, maim themselves, -since these "philes" have not been checked for accuracy, -are not the product of professional experience, -and are often highly fanciful. But the gloating menace -of these philes is not to be entirely dismissed. - -Hackers may not be "serious" about bombing; if they were, -we would hear far more about exploding flashlights, homemade bazookas, -and gym teachers poisoned by chlorine and potassium. -However, hackers are VERY serious about forbidden knowledge. -They are possessed not merely by curiosity, but by -a positive LUST TO KNOW. The desire to know what -others don't is scarcely new. But the INTENSITY -of this desire, as manifested by these young technophilic -denizens of the Information Age, may in fact BE new, -and may represent some basic shift in social values-- -a harbinger of what the world may come to, as society -lays more and more value on the possession, -assimilation and retailing of INFORMATION -as a basic commodity of daily life. - -There have always been young men with obsessive interests -in these topics. Never before, however, have they been able -to network so extensively and easily, and to propagandize -their interests with impunity to random passers-by. -High-school teachers will recognize that there's always -one in a crowd, but when the one in a crowd escapes control -by jumping into the phone-lines, and becomes a hundred such kids -all together on a board, then trouble is brewing visibly. -The urge of authority to DO SOMETHING, even something drastic, -is hard to resist. And in 1990, authority did something. -In fact authority did a great deal. - -# - -The process by which boards create hackers goes something -like this. A youngster becomes interested in computers-- -usually, computer games. He hears from friends that -"bulletin boards" exist where games can be obtained for free. -(Many computer games are "freeware," not copyrighted-- -invented simply for the love of it and given away to the public; -some of these games are quite good.) He bugs his parents for a modem, -or quite often, uses his parents' modem. - -The world of boards suddenly opens up. Computer games -can be quite expensive, real budget-breakers for a kid, -but pirated games, stripped of copy protection, are cheap or free. -They are also illegal, but it is very rare, almost unheard of, -for a small-scale software pirate to be prosecuted. -Once "cracked" of its copy protection, the program, -being digital data, becomes infinitely reproducible. -Even the instructions to the game, any manuals that accompany it, -can be reproduced as text files, or photocopied from legitimate sets. -Other users on boards can give many useful hints in game-playing tactics. -And a youngster with an infinite supply of free computer games can -certainly cut quite a swath among his modem-less friends. - -And boards are pseudonymous. No one need know that you're -fourteen years old--with a little practice at subterfuge, -you can talk to adults about adult things, and be accepted -and taken seriously! You can even pretend to be a girl, -or an old man, or anybody you can imagine. If you find this -kind of deception gratifying, there is ample opportunity -to hone your ability on boards. - -But local boards can grow stale. And almost every board maintains -a list of phone-numbers to other boards, some in distant, tempting, -exotic locales. Who knows what they're up to, in Oregon or Alaska -or Florida or California? It's very easy to find out--just order -the modem to call through its software--nothing to this, just typing -on a keyboard, the same thing you would do for most any computer game. -The machine reacts swiftly and in a few seconds you are talking to -a bunch of interesting people on another seaboard. - -And yet the BILLS for this trivial action can be staggering! -Just by going tippety-tap with your fingers, you may have -saddled your parents with four hundred bucks in long-distance charges, -and gotten chewed out but good. That hardly seems fair. - -How horrifying to have made friends in another state -and to be deprived of their company--and their software-- -just because telephone companies demand absurd amounts of money! -How painful, to be restricted to boards in one's own AREA CODE-- -what the heck is an "area code" anyway, and what makes it so special? -A few grumbles, complaints, and innocent questions of this sort -will often elicit a sympathetic reply from another board user-- -someone with some stolen codes to hand. You dither a while, -knowing this isn't quite right, then you make up your mind -to try them anyhow--AND THEY WORK! Suddenly you're doing something -even your parents can't do. Six months ago you were just some kid--now, -you're the Crimson Flash of Area Code 512! You're bad--you're nationwide! - -Maybe you'll stop at a few abused codes. Maybe you'll decide that -boards aren't all that interesting after all, that it's wrong, -not worth the risk --but maybe you won't. The next step -is to pick up your own repeat-dialling program-- -to learn to generate your own stolen codes. -(This was dead easy five years ago, much harder -to get away with nowadays, but not yet impossible.) -And these dialling programs are not complex or intimidating-- -some are as small as twenty lines of software. - -Now, you too can share codes. You can trade codes to learn -other techniques. If you're smart enough to catch on, -and obsessive enough to want to bother, and ruthless enough -to start seriously bending rules, then you'll get better, fast. -You start to develop a rep. You move up to a heavier class -of board--a board with a bad attitude, the kind of board -that naive dopes like your classmates and your former self -have never even heard of! You pick up the jargon of phreaking -and hacking from the board. You read a few of those anarchy philes-- -and man, you never realized you could be a real OUTLAW without -ever leaving your bedroom. - -You still play other computer games, but now you have a new -and bigger game. This one will bring you a different kind of status -than destroying even eight zillion lousy space invaders. - -Hacking is perceived by hackers as a "game." This is -not an entirely unreasonable or sociopathic perception. -You can win or lose at hacking, succeed or fail, -but it never feels "real." It's not simply that -imaginative youngsters sometimes have a hard time -telling "make-believe" from "real life." Cyberspace -is NOT REAL! "Real" things are physical objects -like trees and shoes and cars. Hacking takes place -on a screen. Words aren't physical, numbers -(even telephone numbers and credit card numbers) -aren't physical. Sticks and stones may break my bones, -but data will never hurt me. Computers SIMULATE reality, -like computer games that simulate tank battles or dogfights -or spaceships. Simulations are just make-believe, -and the stuff in computers is NOT REAL. - -Consider this: if "hacking" is supposed to be so serious and -real-life and dangerous, then how come NINE-YEAR-OLD KIDS have -computers and modems? You wouldn't give a nine year old his own car, -or his own rifle, or his own chainsaw--those things are "real." - -People underground are perfectly aware that the "game" -is frowned upon by the powers that be. Word gets around -about busts in the underground. Publicizing busts is one -of the primary functions of pirate boards, but they also -promulgate an attitude about them, and their own idiosyncratic -ideas of justice. The users of underground boards won't complain -if some guy is busted for crashing systems, spreading viruses, -or stealing money by wire-fraud. They may shake their heads -with a sneaky grin, but they won't openly defend these practices. -But when a kid is charged with some theoretical amount of theft: -$233,846.14, for instance, because he sneaked into a computer -and copied something, and kept it in his house on a floppy disk-- -this is regarded as a sign of near-insanity from prosecutors, -a sign that they've drastically mistaken the immaterial game -of computing for their real and boring everyday world -of fatcat corporate money. - -It's as if big companies and their suck-up lawyers -think that computing belongs to them, and they can -retail it with price stickers, as if it were boxes -of laundry soap! But pricing "information" is like -trying to price air or price dreams. Well, anybody -on a pirate board knows that computing can be, -and ought to be, FREE. Pirate boards are little -independent worlds in cyberspace, and they don't belong -to anybody but the underground. Underground boards -aren't "brought to you by Procter & Gamble." - -To log on to an underground board can mean to -experience liberation, to enter a world where, -for once, money isn't everything and adults -don't have all the answers. - -Let's sample another vivid hacker manifesto. Here are -some excerpts from "The Conscience of a Hacker," by "The Mentor," -from Phrack Volume One, Issue 7, Phile 3. - -"I made a discovery today. I found a computer. -Wait a second, this is cool. It does what I want it to. -If it makes a mistake, it's because I screwed it up. -Not because it doesn't like me. (. . .) -"And then it happened. . .a door opened to a world. . . -rushing through the phone line like heroin through an -addict's veins, an electronic pulse is sent out, -a refuge from day-to-day incompetencies is sought. . . -a board is found. `This is it. . .this is where I belong. . .' -"I know everyone here. . .even if I've never met them, -never talked to them, may never hear from them again. . . -I know you all. . . (. . .) - -"This is our world now. . .the world of the electron -and the switch, the beauty of the baud. We make use of a -service already existing without paying for what could be -dirt-cheap if it wasn't run by profiteering gluttons, and you -call us criminals. We explore. . .and you call us criminals. -We seek after knowledge. . .and you call us criminals. -We exist without skin color, without nationality, -without religious bias. . .and you call us criminals. -You build atomic bombs, you wage wars, you murder, -cheat and lie to us and try to make us believe that -it's for our own good, yet we're the criminals. - -"Yes, I am a criminal. My crime is that of curiosity. -My crime is that of judging people by what they say and think, -not what they look like. My crime is that of outsmarting you, -something that you will never forgive me for." - -# - -There have been underground boards almost as long -as there have been boards. One of the first was 8BBS, -which became a stronghold of the West Coast phone-phreak elite. -After going on-line in March 1980, 8BBS sponsored "Susan Thunder," -and "Tuc," and, most notoriously, "the Condor." "The Condor" -bore the singular distinction of becoming the most vilified -American phreak and hacker ever. Angry underground associates, -fed up with Condor's peevish behavior, turned him in to police, -along with a heaping double-helping of outrageous hacker legendry. -As a result, Condor was kept in solitary confinement for seven months, -for fear that he might start World War Three by triggering missile silos -from the prison payphone. (Having served his time, Condor is now -walking around loose; WWIII has thus far conspicuously failed to occur.) - -The sysop of 8BBS was an ardent free-speech enthusiast -who simply felt that ANY attempt to restrict the expression -of his users was unconstitutional and immoral. -Swarms of the technically curious entered 8BBS -and emerged as phreaks and hackers, until, in 1982, -a friendly 8BBS alumnus passed the sysop a new modem -which had been purchased by credit-card fraud. -Police took this opportunity to seize the entire board -and remove what they considered an attractive nuisance. - -Plovernet was a powerful East Coast pirate board -that operated in both New York and Florida. -Owned and operated by teenage hacker "Quasi Moto," -Plovernet attracted five hundred eager users in 1983. -"Emmanuel Goldstein" was one-time co-sysop of Plovernet, -along with "Lex Luthor," founder of the "Legion of Doom" group. -Plovernet bore the signal honor of being the original home -of the "Legion of Doom," about which the reader will be hearing -a great deal, soon. - -"Pirate-80," or "P-80," run by a sysop known as "Scan-Man," -got into the game very early in Charleston, and continued -steadily for years. P-80 flourished so flagrantly that -even its most hardened users became nervous, and some -slanderously speculated that "Scan Man" must have ties -to corporate security, a charge he vigorously denied. - -"414 Private" was the home board for the first GROUP -to attract conspicuous trouble, the teenage "414 Gang," -whose intrusions into Sloan-Kettering Cancer Center and -Los Alamos military computers were to be a nine-days-wonder in 1982. - -At about this time, the first software piracy boards -began to open up, trading cracked games for the Atari 800 -and the Commodore C64. Naturally these boards were -heavily frequented by teenagers. And with the 1983 -release of the hacker-thriller movie War Games, -the scene exploded. It seemed that every kid -in America had demanded and gotten a modem for Christmas. -Most of these dabbler wannabes put their modems in the attic -after a few weeks, and most of the remainder minded their -P's and Q's and stayed well out of hot water. But some -stubborn and talented diehards had this hacker kid in -War Games figured for a happening dude. They simply -could not rest until they had contacted the underground-- -or, failing that, created their own. - -In the mid-80s, underground boards sprang up like digital fungi. -ShadowSpawn Elite. Sherwood Forest I, II, and III. -Digital Logic Data Service in Florida, sysoped by no less -a man than "Digital Logic" himself; Lex Luthor of the -Legion of Doom was prominent on this board, since it -was in his area code. Lex's own board, "Legion of Doom," -started in 1984. The Neon Knights ran a network of Apple- -hacker boards: Neon Knights North, South, East and West. -Free World II was run by "Major Havoc." Lunatic Labs -is still in operation as of this writing. Dr. Ripco -in Chicago, an anything-goes anarchist board with an -extensive and raucous history, was seized by Secret Service -agents in 1990 on Sundevil day, but up again almost immediately, -with new machines and scarcely diminished vigor. - -The St. Louis scene was not to rank with major centers -of American hacking such as New York and L.A. But St. -Louis did rejoice in possession of "Knight Lightning" -and "Taran King," two of the foremost JOURNALISTS native -to the underground. Missouri boards like Metal Shop, -Metal Shop Private, Metal Shop Brewery, may not have -been the heaviest boards around in terms of illicit -expertise. But they became boards where hackers could -exchange social gossip and try to figure out what the -heck was going on nationally--and internationally. -Gossip from Metal Shop was put into the form of news files, -then assembled into a general electronic publication, -Phrack, a portmanteau title coined from "phreak" and "hack." -The Phrack editors were as obsessively curious about other -hackers as hackers were about machines. - -Phrack, being free of charge and lively reading, began -to circulate throughout the underground. As Taran King -and Knight Lightning left high school for college, -Phrack began to appear on mainframe machines linked to BITNET, -and, through BITNET to the "Internet," that loose but -extremely potent not-for-profit network where academic, -governmental and corporate machines trade data through -the UNIX TCP/IP protocol. (The "Internet Worm" of -November 2-3,1988, created by Cornell grad student Robert Morris, -was to be the largest and best-publicized computer-intrusion scandal -to date. Morris claimed that his ingenious "worm" program was meant -to harmlessly explore the Internet, but due to bad programming, -the Worm replicated out of control and crashed some six thousand -Internet computers. Smaller-scale and less ambitious Internet hacking -was a standard for the underground elite.) - -Most any underground board not hopelessly lame and out-of-it -would feature a complete run of Phrack--and, possibly, -the lesser-known standards of the underground: -the Legion of Doom Technical Journal, the obscene -and raucous Cult of the Dead Cow files, P/HUN magazine, -Pirate, the Syndicate Reports, and perhaps the highly -anarcho-political Activist Times Incorporated. - -Possession of Phrack on one's board was prima facie -evidence of a bad attitude. Phrack was seemingly everywhere, -aiding, abetting, and spreading the underground ethos. -And this did not escape the attention of corporate security -or the police. - -We now come to the touchy subject of police and boards. -Police, do, in fact, own boards. In 1989, there were -police-sponsored boards in California, Colorado, Florida, -Georgia, Idaho, Michigan, Missouri, Texas, and Virginia: -boards such as "Crime Bytes," "Crimestoppers," "All Points" -and "Bullet-N-Board." Police officers, as private computer -enthusiasts, ran their own boards in Arizona, California, -Colorado, Connecticut, Florida, Missouri, Maryland, -New Mexico, North Carolina, Ohio, Tennessee and Texas. -Police boards have often proved helpful in community relations. -Sometimes crimes are reported on police boards. - -Sometimes crimes are COMMITTED on police boards. -This has sometimes happened by accident, as naive hackers -blunder onto police boards and blithely begin offering telephone codes. -Far more often, however, it occurs through the now almost-traditional -use of "sting boards." The first police sting-boards were established -in 1985: "Underground Tunnel" in Austin, Texas, whose sysop -Sgt. Robert Ansley called himself "Pluto"--"The Phone Company" -in Phoenix, Arizona, run by Ken MacLeod of the Maricopa County -Sheriff's office--and Sgt. Dan Pasquale's board in Fremont, California. -Sysops posed as hackers, and swiftly garnered coteries of ardent users, -who posted codes and loaded pirate software with abandon, -and came to a sticky end. - -Sting boards, like other boards, are cheap to operate, -very cheap by the standards of undercover police operations. -Once accepted by the local underground, sysops will likely be -invited into other pirate boards, where they can compile more dossiers. -And when the sting is announced and the worst offenders arrested, -the publicity is generally gratifying. The resultant paranoia -in the underground--perhaps more justly described as a "deterrence effect"-- -tends to quell local lawbreaking for quite a while. - -Obviously police do not have to beat the underbrush for hackers. -On the contrary, they can go trolling for them. Those caught -can be grilled. Some become useful informants. They can lead -the way to pirate boards all across the country. - -And boards all across the country showed the sticky -fingerprints of Phrack, and of that loudest and most -flagrant of all underground groups, the "Legion of Doom." - -The term "Legion of Doom" came from comic books. The Legion of Doom, -a conspiracy of costumed super- villains headed by the chrome-domed -criminal ultra- mastermind Lex Luthor, gave Superman a lot of four-color -graphic trouble for a number of decades. Of course, Superman, -that exemplar of Truth, Justice, and the American Way, -always won in the long run. This didn't matter to the hacker Doomsters-- -"Legion of Doom" was not some thunderous and evil Satanic reference, -it was not meant to be taken seriously. "Legion of Doom" came -from funny-books and was supposed to be funny. - -"Legion of Doom" did have a good mouthfilling ring to it, though. -It sounded really cool. Other groups, such as the "Farmers of Doom," -closely allied to LoD, recognized this grandiloquent quality, -and made fun of it. There was even a hacker group called -"Justice League of America," named after Superman's club -of true-blue crimefighting superheros. - -But they didn't last; the Legion did. - -The original Legion of Doom, hanging out on Quasi Moto's Plovernet board, -were phone phreaks. They weren't much into computers. "Lex Luthor" himself -(who was under eighteen when he formed the Legion) was a COSMOS expert, -COSMOS being the "Central System for Mainframe Operations," -a telco internal computer network. Lex would eventually become -quite a dab hand at breaking into IBM mainframes, but although -everyone liked Lex and admired his attitude, he was not considered -a truly accomplished computer intruder. Nor was he the "mastermind" -of the Legion of Doom--LoD were never big on formal leadership. -As a regular on Plovernet and sysop of his "Legion of Doom BBS," -Lex was the Legion's cheerleader and recruiting officer. - -Legion of Doom began on the ruins of an earlier phreak group, -The Knights of Shadow. Later, LoD was to subsume the personnel -of the hacker group "Tribunal of Knowledge." People came and went -constantly in LoD; groups split up or formed offshoots. - -Early on, the LoD phreaks befriended a few computer-intrusion -enthusiasts, who became the associated "Legion of Hackers." -Then the two groups conflated into the "Legion of Doom/Hackers," -or LoD/H. When the original "hacker" wing, Messrs. "Compu-Phreak" -and "Phucked Agent 04," found other matters to occupy their time, -the extra "/H" slowly atrophied out of the name; but by this time -the phreak wing, Messrs. Lex Luthor, "Blue Archer," "Gary Seven," -"Kerrang Khan," "Master of Impact," "Silver Spy," "The Marauder," -and "The Videosmith," had picked up a plethora of intrusion -expertise and had become a force to be reckoned with. - -LoD members seemed to have an instinctive understanding -that the way to real power in the underground lay through -covert publicity. LoD were flagrant. Not only was it one -of the earliest groups, but the members took pains to widely -distribute their illicit knowledge. Some LoD members, -like "The Mentor," were close to evangelical about it. -Legion of Doom Technical Journal began to show up on boards -throughout the underground. - -LoD Technical Journal was named in cruel parody -of the ancient and honored AT&T Technical Journal. -The material in these two publications was quite similar-- -much of it, adopted from public journals and discussions -in the telco community. And yet, the predatory attitude -of LoD made even its most innocuous data seem deeply sinister; -an outrage; a clear and present danger. - -To see why this should be, let's consider the following -(invented) paragraphs, as a kind of thought experiment. - -(A) "W. Fred Brown, AT&T Vice President for -Advanced Technical Development, testified May 8 -at a Washington hearing of the National Telecommunications -and Information Administration (NTIA), regarding -Bellcore's GARDEN project. GARDEN (Generalized -Automatic Remote Distributed Electronic Network) is a -telephone-switch programming tool that makes it possible -to develop new telecom services, including hold-on-hold -and customized message transfers, from any keypad terminal, -within seconds. The GARDEN prototype combines centrex -lines with a minicomputer using UNIX operating system software." - -(B) "Crimson Flash 512 of the Centrex Mobsters reports: -D00dz, you wouldn't believe this GARDEN bullshit Bellcore's -just come up with! Now you don't even need a lousy Commodore -to reprogram a switch--just log on to GARDEN as a technician, -and you can reprogram switches right off the keypad in any -public phone booth! You can give yourself hold-on-hold -and customized message transfers, and best of all, -the thing is run off (notoriously insecure) centrex lines -using--get this--standard UNIX software! Ha ha ha ha!" - -Message (A), couched in typical techno-bureaucratese, -appears tedious and almost unreadable. (A) scarcely seems -threatening or menacing. Message (B), on the other hand, -is a dreadful thing, prima facie evidence of a dire conspiracy, -definitely not the kind of thing you want your teenager reading. - -The INFORMATION, however, is identical. It is PUBLIC -information, presented before the federal government in -an open hearing. It is not "secret." It is not "proprietary." -It is not even "confidential." On the contrary, the -development of advanced software systems is a matter -of great public pride to Bellcore. - -However, when Bellcore publicly announces a project of this kind, -it expects a certain attitude from the public--something along -the lines of GOSH WOW, YOU GUYS ARE GREAT, KEEP THAT UP, WHATEVER IT IS-- -certainly not cruel mimickry, one-upmanship and outrageous speculations -about possible security holes. - -Now put yourself in the place of a policeman confronted by -an outraged parent, or telco official, with a copy of Version (B). -This well-meaning citizen, to his horror, has discovered -a local bulletin-board carrying outrageous stuff like (B), -which his son is examining with a deep and unhealthy interest. -If (B) were printed in a book or magazine, you, as an American -law enforcement officer, would know that it would take -a hell of a lot of trouble to do anything about it; -but it doesn't take technical genius to recognize that -if there's a computer in your area harboring stuff like (B), -there's going to be trouble. - -In fact, if you ask around, any computer-literate cop -will tell you straight out that boards with stuff like (B) -are the SOURCE of trouble. And the WORST source of trouble -on boards are the ringleaders inventing and spreading stuff like (B). -If it weren't for these jokers, there wouldn't BE any trouble. - -And Legion of Doom were on boards like nobody else. -Plovernet. The Legion of Doom Board. The Farmers of Doom Board. -Metal Shop. OSUNY. Blottoland. Private Sector. Atlantis. -Digital Logic. Hell Phrozen Over. - -LoD members also ran their own boards. "Silver Spy" started -his own board, "Catch-22," considered one of the heaviest around. -So did "Mentor," with his "Phoenix Project." When they didn't run boards -themselves, they showed up on other people's boards, to brag, boast, -and strut. And where they themselves didn't go, their philes went, -carrying evil knowledge and an even more evil attitude. - -As early as 1986, the police were under the vague impression -that EVERYONE in the underground was Legion of Doom. -LoD was never that large--considerably smaller than either -"Metal Communications" or "The Administration," for instance-- -but LoD got tremendous press. Especially in Phrack, -which at times read like an LoD fan magazine; and Phrack -was everywhere, especially in the offices of telco security. -You couldn't GET busted as a phone phreak, a hacker, -or even a lousy codes kid or warez dood, without the cops -asking if you were LoD. - -This was a difficult charge to deny, as LoD never -distributed membership badges or laminated ID cards. -If they had, they would likely have died out quickly, -for turnover in their membership was considerable. -LoD was less a high-tech street-gang than an ongoing -state-of-mind. LoD was the Gang That Refused to Die. -By 1990, LoD had RULED for ten years, and it seemed WEIRD -to police that they were continually busting people who were -only sixteen years old. All these teenage small-timers -were pleading the tiresome hacker litany of "just curious, -no criminal intent." Somewhere at the center of this -conspiracy there had to be some serious adult masterminds, -not this seemingly endless supply of myopic suburban -white kids with high SATs and funny haircuts. - -There was no question that most any American hacker -arrested would "know" LoD. They knew the handles -of contributors to LoD Tech Journal, and were likely -to have learned their craft through LoD boards and LoD activism. -But they'd never met anyone from LoD. Even some of the -rotating cadre who were actually and formally "in LoD" -knew one another only by board-mail and pseudonyms. -This was a highly unconventional profile for a criminal conspiracy. -Computer networking, and the rapid evolution of the digital underground, -made the situation very diffuse and confusing. - -Furthermore, a big reputation in the digital underground -did not coincide with one's willingness to commit "crimes." -Instead, reputation was based on cleverness and technical mastery. -As a result, it often seemed that the HEAVIER the hackers were, -the LESS likely they were to have committed any kind of common, -easily prosecutable crime. There were some hackers who could really steal. -And there were hackers who could really hack. But the two groups didn't seem -to overlap much, if at all. For instance, most people in the underground -looked up to "Emmanuel Goldstein" of 2600 as a hacker demigod. -But Goldstein's publishing activities were entirely legal-- -Goldstein just printed dodgy stuff and talked about politics, -he didn't even hack. When you came right down to it, -Goldstein spent half his time complaining that computer security -WASN'T STRONG ENOUGH and ought to be drastically improved -across the board! - -Truly heavy-duty hackers, those with serious technical skills -who had earned the respect of the underground, never stole money -or abused credit cards. Sometimes they might abuse phone-codes-- -but often, they seemed to get all the free phone-time they wanted -without leaving a trace of any kind. - -The best hackers, the most powerful and technically accomplished, -were not professional fraudsters. They raided computers habitually, -but wouldn't alter anything, or damage anything. They didn't even steal -computer equipment--most had day-jobs messing with hardware, -and could get all the cheap secondhand equipment they wanted. -The hottest hackers, unlike the teenage wannabes, weren't snobs -about fancy or expensive hardware. Their machines tended to be -raw second-hand digital hot-rods full of custom add-ons that -they'd cobbled together out of chickenwire, memory chips and spit. -Some were adults, computer software writers and consultants by trade, -and making quite good livings at it. Some of them ACTUALLY WORKED -FOR THE PHONE COMPANY--and for those, the "hackers" actually found -under the skirts of Ma Bell, there would be little mercy in 1990. - -It has long been an article of faith in the -underground that the "best" hackers never get caught. -They're far too smart, supposedly. They never get caught -because they never boast, brag, or strut. These demigods -may read underground boards (with a condescending smile), -but they never say anything there. The "best" hackers, -according to legend, are adult computer professionals, -such as mainframe system administrators, who already know -the ins and outs of their particular brand of security. -Even the "best" hacker can't break in to just any computer at random: -the knowledge of security holes is too specialized, varying widely -with different software and hardware. But if people are employed to run, -say, a UNIX mainframe or a VAX/VMS machine, then they tend to learn -security from the inside out. Armed with this knowledge, -they can look into most anybody else's UNIX or VMS -without much trouble or risk, if they want to. -And, according to hacker legend, of course they want to, -so of course they do. They just don't make a big deal -of what they've done. So nobody ever finds out. - -It is also an article of faith in the underground that -professional telco people "phreak" like crazed weasels. -OF COURSE they spy on Madonna's phone calls--I mean, -WOULDN'T YOU? Of course they give themselves free long- -distance--why the hell should THEY pay, they're running -the whole shebang! - -It has, as a third matter, long been an article of faith -that any hacker caught can escape serious punishment if -he confesses HOW HE DID IT. Hackers seem to believe -that governmental agencies and large corporations are -blundering about in cyberspace like eyeless jellyfish -or cave salamanders. They feel that these large -but pathetically stupid organizations will proffer up -genuine gratitude, and perhaps even a security post -and a big salary, to the hot-shot intruder who will deign -to reveal to them the supreme genius of his modus operandi. - -In the case of longtime LoD member "Control-C," -this actually happened, more or less. Control-C had led -Michigan Bell a merry chase, and when captured in 1987, -he turned out to be a bright and apparently physically -harmless young fanatic, fascinated by phones. There was -no chance in hell that Control-C would actually repay the -enormous and largely theoretical sums in long-distance -service that he had accumulated from Michigan Bell. -He could always be indicted for fraud or computer-intrusion, -but there seemed little real point in this--he hadn't -physically damaged any computer. He'd just plead guilty, -and he'd likely get the usual slap-on-the-wrist, -and in the meantime it would be a big hassle for Michigan Bell -just to bring up the case. But if kept on the payroll, -he might at least keep his fellow hackers at bay. - -There were uses for him. For instance, a contrite -Control-C was featured on Michigan Bell internal posters, -sternly warning employees to shred their trash. -He'd always gotten most of his best inside info from -"trashing"--raiding telco dumpsters, for useful data -indiscreetly thrown away. He signed these posters, too. -Control-C had become something like a Michigan Bell mascot. -And in fact, Control-C DID keep other hackers at bay. -Little hackers were quite scared of Control-C and his -heavy-duty Legion of Doom friends. And big hackers WERE -his friends and didn't want to screw up his cushy situation. - -No matter what one might say of LoD, they did stick together. -When "Wasp," an apparently genuinely malicious New York hacker, -began crashing Bellcore machines, Control-C received swift volunteer -help from "the Mentor" and the Georgia LoD wing made up of -"The Prophet," "Urvile," and "Leftist." Using Mentor's Phoenix -Project board to coordinate, the Doomsters helped telco security -to trap Wasp, by luring him into a machine with a tap -and line-trace installed. Wasp lost. LoD won! And my, did they brag. - -Urvile, Prophet and Leftist were well-qualified for this activity, -probably more so even than the quite accomplished Control-C. -The Georgia boys knew all about phone switching-stations. -Though relative johnny-come-latelies in the Legion of Doom, -they were considered some of LoD's heaviest guys, -into the hairiest systems around. They had the good fortune -to live in or near Atlanta, home of the sleepy and apparently -tolerant BellSouth RBOC. - -As RBOC security went, BellSouth were "cake." US West (of Arizona, -the Rockies and the Pacific Northwest) were tough and aggressive, -probably the heaviest RBOC around. Pacific Bell, California's PacBell, -were sleek, high-tech, and longtime veterans of the LA phone-phreak wars. -NYNEX had the misfortune to run the New York City area, and were warily -prepared for most anything. Even Michigan Bell, a division of the -Ameritech RBOC, at least had the elementary sense to hire their own hacker -as a useful scarecrow. But BellSouth, even though their corporate P.R. -proclaimed them to have "Everything You Expect From a Leader," were pathetic. - -When rumor about LoD's mastery of Georgia's switching network got around -to BellSouth through Bellcore and telco security scuttlebutt, -they at first refused to believe it. If you paid serious attention -to every rumor out and about these hacker kids, you would hear all kinds -of wacko saucer-nut nonsense: that the National Security Agency -monitored all American phone calls, that the CIA and DEA tracked -traffic on bulletin-boards with word-analysis programs, -that the Condor could start World War III from a payphone. - -If there were hackers into BellSouth switching-stations, then how come -nothing had happened? Nothing had been hurt. BellSouth's machines -weren't crashing. BellSouth wasn't suffering especially badly from fraud. -BellSouth's customers weren't complaining. BellSouth was headquartered -in Atlanta, ambitious metropolis of the new high-tech Sunbelt; -and BellSouth was upgrading its network by leaps and bounds, -digitizing the works left right and center. They could hardly be -considered sluggish or naive. BellSouth's technical expertise -was second to none, thank you kindly. But then came the Florida business. - -On June 13, 1989, callers to the Palm Beach County Probation Department, -in Delray Beach, Florida, found themselves involved in a remarkable -discussion with a phone-sex worker named "Tina" in New York State. -Somehow, ANY call to this probation office near Miami was instantly -and magically transported across state lines, at no extra charge to the user, -to a pornographic phone-sex hotline hundreds of miles away! - -This practical joke may seem utterly hilarious at first hearing, -and indeed there was a good deal of chuckling about it in -phone phreak circles, including the Autumn 1989 issue of 2600. -But for Southern Bell (the division of the BellSouth RBOC -supplying local service for Florida, Georgia, North Carolina -and South Carolina), this was a smoking gun. For the first time ever, -a computer intruder had broken into a BellSouth central office -switching station and re-programmed it! - -Or so BellSouth thought in June 1989. Actually, LoD members had been -frolicking harmlessly in BellSouth switches since September 1987. -The stunt of June 13--call-forwarding a number through manipulation -of a switching station--was child's play for hackers as accomplished -as the Georgia wing of LoD. Switching calls interstate sounded like -a big deal, but it took only four lines of code to accomplish this. -An easy, yet more discreet, stunt, would be to call-forward another -number to your own house. If you were careful and considerate, -and changed the software back later, then not a soul would know. -Except you. And whoever you had bragged to about it. - -As for BellSouth, what they didn't know wouldn't hurt them. - -Except now somebody had blown the whole thing wide open, and BellSouth knew. - -A now alerted and considerably paranoid BellSouth began searching switches -right and left for signs of impropriety, in that hot summer of 1989. -No fewer than forty-two BellSouth employees were put on 12-hour shifts, -twenty-four hours a day, for two solid months, poring over records -and monitoring computers for any sign of phony access. These forty-two -overworked experts were known as BellSouth's "Intrusion Task Force." - -What the investigators found astounded them. Proprietary telco databases -had been manipulated: phone numbers had been created out of thin air, -with no users' names and no addresses. And perhaps worst of all, -no charges and no records of use. The new digital ReMOB (Remote Observation) -diagnostic feature had been extensively tampered with--hackers had learned to -reprogram ReMOB software, so that they could listen in on any switch-routed -call at their leisure! They were using telco property to SPY! - -The electrifying news went out throughout law enforcement in 1989. -It had never really occurred to anyone at BellSouth that their prized -and brand-new digital switching-stations could be RE-PROGRAMMED. -People seemed utterly amazed that anyone could have the nerve. -Of course these switching stations were "computers," and everybody -knew hackers liked to "break into computers:" but telephone people's -computers were DIFFERENT from normal people's computers. - -The exact reason WHY these computers were "different" was -rather ill-defined. It certainly wasn't the extent of their security. -The security on these BellSouth computers was lousy; the AIMSX computers, -for instance, didn't even have passwords. But there was no question that -BellSouth strongly FELT that their computers were very different indeed. -And if there were some criminals out there who had not gotten that message, -BellSouth was determined to see that message taught. - -After all, a 5ESS switching station was no mere bookkeeping system for -some local chain of florists. Public service depended on these stations. -Public SAFETY depended on these stations. - -And hackers, lurking in there call-forwarding or ReMobbing, could spy -on anybody in the local area! They could spy on telco officials! -They could spy on police stations! They could spy on local offices -of the Secret Service. . . . - -In 1989, electronic cops and hacker-trackers began using scrambler-phones -and secured lines. It only made sense. There was no telling who was into -those systems. Whoever they were, they sounded scary. This was some -new level of antisocial daring. Could be West German hackers, in the pay -of the KGB. That too had seemed a weird and farfetched notion, -until Clifford Stoll had poked and prodded a sluggish Washington -law-enforcement bureaucracy into investigating a computer intrusion -that turned out to be exactly that--HACKERS, IN THE PAY OF THE KGB! -Stoll, the systems manager for an Internet lab in Berkeley California, -had ended up on the front page of the New Nork Times, proclaimed a national -hero in the first true story of international computer espionage. -Stoll's counterspy efforts, which he related in a bestselling book, -The Cuckoo's Egg, in 1989, had established the credibility of `hacking' -as a possible threat to national security. The United States Secret Service -doesn't mess around when it suspects a possible action by a foreign -intelligence apparat. - -The Secret Service scrambler-phones and secured lines put -a tremendous kink in law enforcement's ability to operate freely; -to get the word out, cooperate, prevent misunderstandings. -Nevertheless, 1989 scarcely seemed the time for half-measures. -If the police and Secret Service themselves were not operationally secure, -then how could they reasonably demand measures of security from -private enterprise? At least, the inconvenience made people aware -of the seriousness of the threat. - -If there was a final spur needed to get the police off the dime, -it came in the realization that the emergency 911 system was vulnerable. -The 911 system has its own specialized software, but it is run on the same -digital switching systems as the rest of the telephone network. -911 is not physically different from normal telephony. But it is -certainly culturally different, because this is the area of -telephonic cyberspace reserved for the police and emergency services. - -Your average policeman may not know much about hackers or phone-phreaks. -Computer people are weird; even computer COPS are rather weird; -the stuff they do is hard to figure out. But a threat to the 911 system -is anything but an abstract threat. If the 911 system goes, people can die. - -Imagine being in a car-wreck, staggering to a phone-booth, -punching 911 and hearing "Tina" pick up the phone-sex line -somewhere in New York! The situation's no longer comical, somehow. - -And was it possible? No question. Hackers had attacked 911 -systems before. Phreaks can max-out 911 systems just by siccing -a bunch of computer-modems on them in tandem, dialling them over -and over until they clog. That's very crude and low-tech, -but it's still a serious business. - -The time had come for action. It was time to take stern measures -with the underground. It was time to start picking up the dropped threads, -the loose edges, the bits of braggadocio here and there; it was time to get -on the stick and start putting serious casework together. Hackers weren't -"invisible." They THOUGHT they were invisible; but the truth was, -they had just been tolerated too long. - -Under sustained police attention in the summer of '89, the digital -underground began to unravel as never before. - -The first big break in the case came very early on: July 1989, -the following month. The perpetrator of the "Tina" switch was caught, -and confessed. His name was "Fry Guy," a 16-year-old in Indiana. -Fry Guy had been a very wicked young man. - -Fry Guy had earned his handle from a stunt involving French fries. -Fry Guy had filched the log-in of a local MacDonald's manager -and had logged-on to the MacDonald's mainframe on the Sprint -Telenet system. Posing as the manager, Fry Guy had altered -MacDonald's records, and given some teenage hamburger-flipping -friends of his, generous raises. He had not been caught. - -Emboldened by success, Fry Guy moved on to credit-card abuse. -Fry Guy was quite an accomplished talker; with a gift for -"social engineering." If you can do "social engineering" ---fast-talk, fake-outs, impersonation, conning, scamming-- -then card abuse comes easy. (Getting away with it in -the long run is another question). - -Fry Guy had run across "Urvile" of the Legion of Doom -on the ALTOS Chat board in Bonn, Germany. ALTOS Chat -was a sophisticated board, accessible through globe-spanning -computer networks like BITnet, Tymnet, and Telenet. -ALTOS was much frequented by members of Germany's -Chaos Computer Club. Two Chaos hackers who hung out on ALTOS, -"Jaeger" and "Pengo," had been the central villains of -Clifford Stoll's Cuckoo's Egg case: consorting in East Berlin -with a spymaster from the KGB, and breaking into American -computers for hire, through the Internet. - -When LoD members learned the story of Jaeger's depredations -from Stoll's book, they were rather less than impressed, -technically speaking. On LoD's own favorite board of the moment, -"Black Ice," LoD members bragged that they themselves could have done -all the Chaos break-ins in a week flat! Nevertheless, LoD were grudgingly -impressed by the Chaos rep, the sheer hairy-eyed daring of hash-smoking -anarchist hackers who had rubbed shoulders with the fearsome big-boys -of international Communist espionage. LoD members sometimes traded -bits of knowledge with friendly German hackers on ALTOS--phone numbers -for vulnerable VAX/VMS computers in Georgia, for instance. -Dutch and British phone phreaks, and the Australian clique of -"Phoenix," "Nom," and "Electron," were ALTOS regulars, too. -In underground circles, to hang out on ALTOS was considered -the sign of an elite dude, a sophisticated hacker of the -international digital jet-set. - -Fry Guy quickly learned how to raid information from credit-card -consumer-reporting agencies. He had over a hundred stolen credit-card -numbers in his notebooks, and upwards of a thousand swiped long-distance -access codes. He knew how to get onto Altos, and how to talk the talk of -the underground convincingly. He now wheedled knowledge of switching-station -tricks from Urvile on the ALTOS system. - -Combining these two forms of knowledge enabled Fry Guy to bootstrap -his way up to a new form of wire-fraud. First, he'd snitched credit card -numbers from credit-company computers. The data he copied included names, -addresses and phone numbers of the random card-holders. - -Then Fry Guy, impersonating a card-holder, called up Western Union -and asked for a cash advance on "his" credit card. Western Union, -as a security guarantee, would call the customer back, at home, -to verify the transaction. - -But, just as he had switched the Florida probation office to "Tina" -in New York, Fry Guy switched the card-holder's number to a local pay-phone. -There he would lurk in wait, muddying his trail by routing and re-routing -the call, through switches as far away as Canada. When the call came through, -he would boldly "social-engineer," or con, the Western Union people, pretending -to be the legitimate card-holder. Since he'd answered the proper phone number, -the deception was not very hard. Western Union's money was then shipped to -a confederate of Fry Guy's in his home town in Indiana. - -Fry Guy and his cohort, using LoD techniques, stole six thousand dollars -from Western Union between December 1988 and July 1989. They also dabbled -in ordering delivery of stolen goods through card-fraud. Fry Guy -was intoxicated with success. The sixteen-year-old fantasized wildly -to hacker rivals, boasting that he'd used rip-off money to hire himself -a big limousine, and had driven out-of-state with a groupie from -his favorite heavy-metal band, Motley Crue. - -Armed with knowledge, power, and a gratifying stream of free money, -Fry Guy now took it upon himself to call local representatives -of Indiana Bell security, to brag, boast, strut, and utter -tormenting warnings that his powerful friends in the notorious -Legion of Doom could crash the national telephone network. -Fry Guy even named a date for the scheme: the Fourth of July, -a national holiday. - -This egregious example of the begging-for-arrest syndrome was shortly -followed by Fry Guy's arrest. After the Indiana telephone company figured -out who he was, the Secret Service had DNRs--Dialed Number Recorders-- -installed on his home phone lines. These devices are not taps, and can't -record the substance of phone calls, but they do record the phone numbers -of all calls going in and out. Tracing these numbers showed Fry Guy's -long-distance code fraud, his extensive ties to pirate bulletin boards, -and numerous personal calls to his LoD friends in Atlanta. By July 11, -1989, Prophet, Urvile and Leftist also had Secret Service DNR -"pen registers" installed on their own lines. - -The Secret Service showed up in force at Fry Guy's house on July 22, 1989, -to the horror of his unsuspecting parents. The raiders were led by -a special agent from the Secret Service's Indianapolis office. -However, the raiders were accompanied and advised by Timothy M. Foley -of the Secret Service's Chicago office (a gentleman about whom -we will soon be hearing a great deal). - -Following federal computer-crime techniques that had been standard -since the early 1980s, the Secret Service searched the house thoroughly, -and seized all of Fry Guy's electronic equipment and notebooks. -All Fry Guy's equipment went out the door in the custody of the -Secret Service, which put a swift end to his depredations. - -The USSS interrogated Fry Guy at length. His case was put in the charge -of Deborah Daniels, the federal US Attorney for the Southern District -of Indiana. Fry Guy was charged with eleven counts of computer fraud, -unauthorized computer access, and wire fraud. The evidence was thorough -and irrefutable. For his part, Fry Guy blamed his corruption on the -Legion of Doom and offered to testify against them. - -Fry Guy insisted that the Legion intended to crash the phone system -on a national holiday. And when AT&T crashed on Martin Luther King Day, -1990, this lent a credence to his claim that genuinely alarmed telco -security and the Secret Service. - -Fry Guy eventually pled guilty on May 31, 1990. On September 14, -he was sentenced to forty-four months' probation and four hundred hours' -community service. He could have had it much worse; but it made sense -to prosecutors to take it easy on this teenage minor, while zeroing -in on the notorious kingpins of the Legion of Doom. - -But the case against LoD had nagging flaws. Despite the best effort -of investigators, it was impossible to prove that the Legion had crashed -the phone system on January 15, because they, in fact, hadn't done so. -The investigations of 1989 did show that certain members of -the Legion of Doom had achieved unprecedented power over the telco -switching stations, and that they were in active conspiracy -to obtain more power yet. Investigators were privately convinced -that the Legion of Doom intended to do awful things with this knowledge, -but mere evil intent was not enough to put them in jail. - -And although the Atlanta Three--Prophet, Leftist, and especially Urvile-- -had taught Fry Guy plenty, they were not themselves credit-card fraudsters. -The only thing they'd "stolen" was long-distance service--and since they'd -done much of that through phone-switch manipulation, there was no easy way -to judge how much they'd "stolen," or whether this practice was even "theft" -of any easily recognizable kind. - -Fry Guy's theft of long-distance codes had cost the phone companies plenty. -The theft of long-distance service may be a fairly theoretical "loss," -but it costs genuine money and genuine time to delete all those stolen codes, -and to re-issue new codes to the innocent owners of those corrupted codes. -The owners of the codes themselves are victimized, and lose time and money -and peace of mind in the hassle. And then there were the credit-card victims -to deal with, too, and Western Union. When it came to rip-off, Fry Guy was -far more of a thief than LoD. It was only when it came to actual computer -expertise that Fry Guy was small potatoes. - -The Atlanta Legion thought most "rules" of cyberspace were for rodents -and losers, but they DID have rules. THEY NEVER CRASHED ANYTHING, -AND THEY NEVER TOOK MONEY. These were rough rules-of-thumb, and -rather dubious principles when it comes to the ethical subtleties -of cyberspace, but they enabled the Atlanta Three to operate with -a relatively clear conscience (though never with peace of mind). - -If you didn't hack for money, if you weren't robbing people of actual funds ---money in the bank, that is-- then nobody REALLY got hurt, in LoD's opinion. -"Theft of service" was a bogus issue, and "intellectual property" was -a bad joke. But LoD had only elitist contempt for rip-off artists, -"leechers," thieves. They considered themselves clean. In their opinion, -if you didn't smash-up or crash any systems --(well, not on purpose, anyhow-- -accidents can happen, just ask Robert Morris) then it was very unfair -to call you a "vandal" or a "cracker." When you were hanging out on-line -with your "pals" in telco security, you could face them down from the higher -plane of hacker morality. And you could mock the police from the supercilious -heights of your hacker's quest for pure knowledge. - -But from the point of view of law enforcement and telco security, however, -Fry Guy was not really dangerous. The Atlanta Three WERE dangerous. -It wasn't the crimes they were committing, but the DANGER, -the potential hazard, the sheer TECHNICAL POWER LoD had accumulated, -that had made the situation untenable. Fry Guy was not LoD. -He'd never laid eyes on anyone in LoD; his only contacts with them -had been electronic. Core members of the Legion of Doom tended to meet -physically for conventions every year or so, to get drunk, give each other -the hacker high-sign, send out for pizza and ravage hotel suites. -Fry Guy had never done any of this. Deborah Daniels assessed Fry Guy -accurately as "an LoD wannabe." - -Nevertheless Fry Guy's crimes would be directly attributed to LoD -in much future police propaganda. LoD would be described as -"a closely knit group" involved in "numerous illegal activities" -including "stealing and modifying individual credit histories," -and "fraudulently obtaining money and property." Fry Guy did this, -but the Atlanta Three didn't; they simply weren't into theft, -but rather intrusion. This caused a strange kink in -the prosecution's strategy. LoD were accused of -"disseminating information about attacking computers -to other computer hackers in an effort to shift the focus -of law enforcement to those other hackers and away from the Legion of Doom." - -This last accusation (taken directly from a press release by the Chicago -Computer Fraud and Abuse Task Force) sounds particularly far-fetched. -One might conclude at this point that investigators would have been -well-advised to go ahead and "shift their focus" from the "Legion of Doom." -Maybe they SHOULD concentrate on "those other hackers"--the ones who were -actually stealing money and physical objects. - -But the Hacker Crackdown of 1990 was not a simple policing action. -It wasn't meant just to walk the beat in cyberspace--it was a CRACKDOWN, -a deliberate attempt to nail the core of the operation, to send a dire -and potent message that would settle the hash of the digital underground -for good. - -By this reasoning, Fry Guy wasn't much more than the electronic equivalent -of a cheap streetcorner dope dealer. As long as the masterminds of LoD were -still flagrantly operating, pushing their mountains of illicit knowledge -right and left, and whipping up enthusiasm for blatant lawbreaking, -then there would be an INFINITE SUPPLY of Fry Guys. - -Because LoD were flagrant, they had left trails everywhere, -to be picked up by law enforcement in New York, Indiana, -Florida, Texas, Arizona, Missouri, even Australia. -But 1990's war on the Legion of Doom was led out of Illinois, -by the Chicago Computer Fraud and Abuse Task Force. - -# - -The Computer Fraud and Abuse Task Force, led by federal prosecutor -William J. Cook, had started in 1987 and had swiftly become one -of the most aggressive local "dedicated computer-crime units." -Chicago was a natural home for such a group. The world's first -computer bulletin-board system had been invented in Illinois. -The state of Illinois had some of the nation's first and sternest -computer crime laws. Illinois State Police were markedly alert -to the possibilities of white-collar crime and electronic fraud. - -And William J. Cook in particular was a rising star in -electronic crime-busting. He and his fellow federal prosecutors -at the U.S. Attorney's office in Chicago had a tight relation -with the Secret Service, especially go-getting Chicago-based agent -Timothy Foley. While Cook and his Department of Justice colleagues -plotted strategy, Foley was their man on the street. - -Throughout the 1980s, the federal government had given prosecutors -an armory of new, untried legal tools against computer crime. -Cook and his colleagues were pioneers in the use of these new statutes -in the real-life cut-and-thrust of the federal courtroom. - -On October 2, 1986, the US Senate had passed the -"Computer Fraud and Abuse Act" unanimously, but there -were pitifully few convictions under this statute. -Cook's group took their name from this statute, -since they were determined to transform this powerful but -rather theoretical Act of Congress into a real-life engine -of legal destruction against computer fraudsters and scofflaws. - -It was not a question of merely discovering crimes, -investigating them, and then trying and punishing their -perpetrators. The Chicago unit, like most everyone else -in the business, already KNEW who the bad guys were: -the Legion of Doom and the writers and editors of Phrack. -The task at hand was to find some legal means of putting -these characters away. - -This approach might seem a bit dubious, to someone not -acquainted with the gritty realities of prosecutorial work. -But prosecutors don't put people in jail for crimes -they have committed; they put people in jail for crimes -they have committed THAT CAN BE PROVED IN COURT. -Chicago federal police put Al Capone in prison -for income-tax fraud. Chicago is a big town, -with a rough-and-ready bare-knuckle tradition -on both sides of the law. - -Fry Guy had broken the case wide open and alerted telco security -to the scope of the problem. But Fry Guy's crimes would not -put the Atlanta Three behind bars--much less the wacko underground -journalists of Phrack. So on July 22, 1989, the same day that -Fry Guy was raided in Indiana, the Secret Service descended upon -the Atlanta Three. - -This was likely inevitable. By the summer of 1989, law enforcement -were closing in on the Atlanta Three from at least six directions at once. -First, there were the leads from Fry Guy, which had led to the DNR registers -being installed on the lines of the Atlanta Three. The DNR evidence alone -would have finished them off, sooner or later. - -But second, the Atlanta lads were already well-known to Control-C -and his telco security sponsors. LoD's contacts with telco security -had made them overconfident and even more boastful than usual; -they felt that they had powerful friends in high places, -and that they were being openly tolerated by telco security. -But BellSouth's Intrusion Task Force were hot on the trail of LoD -and sparing no effort or expense. - -The Atlanta Three had also been identified by name and listed -on the extensive anti-hacker files maintained, and retailed for pay, -by private security operative John Maxfield of Detroit. -Maxfield, who had extensive ties to telco security -and many informants in the underground, was a bete noire -of the Phrack crowd, and the dislike was mutual. - - -The Atlanta Three themselves had written articles for Phrack. -This boastful act could not possibly escape telco and law enforcement -attention. - -"Knightmare," a high-school age hacker from Arizona, -was a close friend and disciple of Atlanta LoD, -but he had been nabbed by the formidable Arizona -Organized Crime and Racketeering Unit. Knightmare was -on some of LoD's favorite boards--"Black Ice" in particular-- -and was privy to their secrets. And to have Gail Thackeray, -the Assistant Attorney General of Arizona, on one's trail -was a dreadful peril for any hacker. - -And perhaps worst of all, Prophet had committed a major blunder -by passing an illicitly copied BellSouth computer-file to Knight Lightning, -who had published it in Phrack. This, as we will see, was an act of dire -consequence for almost everyone concerned. - -On July 22, 1989, the Secret Service showed up at the Leftist's house, -where he lived with his parents. A massive squad of some twenty officers -surrounded the building: Secret Service, federal marshals, local police, -possibly BellSouth telco security; it was hard to tell in the crush. -Leftist's dad, at work in his basement office, first noticed -a muscular stranger in plain clothes crashing through the -back yard with a drawn pistol. As more strangers poured -into the house, Leftist's dad naturally assumed there was -an armed robbery in progress. - -Like most hacker parents, Leftist's mom and dad had only the vaguest -notions of what their son had been up to all this time. Leftist had -a day-job repairing computer hardware. His obsession with computers -seemed a bit odd, but harmless enough, and likely to produce a well- -paying career. The sudden, overwhelming raid left Leftist's -parents traumatized. - -The Leftist himself had been out after work with his co-workers, -surrounding a couple of pitchers of margaritas. As he came trucking -on tequila-numbed feet up the pavement, toting a bag full of floppy-disks, -he noticed a large number of unmarked cars parked in his driveway. -All the cars sported tiny microwave antennas. - -The Secret Service had knocked the front door off its hinges, -almost flattening his mom. - -Inside, Leftist was greeted by Special Agent James Cool -of the US Secret Service, Atlanta office. Leftist was flabbergasted. -He'd never met a Secret Service agent before. He could not imagine -that he'd ever done anything worthy of federal attention. -He'd always figured that if his activities became intolerable, -one of his contacts in telco security would give him a private -phone-call and tell him to knock it off. - -But now Leftist was pat-searched for weapons by grim professionals, -and his bag of floppies was quickly seized. He and his parents were -all shepherded into separate rooms and grilled at length as a score -of officers scoured their home for anything electronic. - -Leftist was horrified as his treasured IBM AT personal computer -with its forty-meg hard disk, and his recently purchased 80386 IBM-clone -with a whopping hundred-meg hard disk, both went swiftly out the door -in Secret Service custody. They also seized all his disks, all his notebooks, -and a tremendous booty in dogeared telco documents that Leftist had snitched -out of trash dumpsters. - -Leftist figured the whole thing for a big misunderstanding. -He'd never been into MILITARY computers. He wasn't a SPY or a COMMUNIST. -He was just a good ol' Georgia hacker, and now he just wanted all these -people out of the house. But it seemed they wouldn't go until he made -some kind of statement. - -And so, he levelled with them. - -And that, Leftist said later from his federal prison camp in Talladega, -Alabama, was a big mistake. The Atlanta area was unique, -in that it had three members of the Legion of Doom who actually -occupied more or less the same physical locality. Unlike the rest -of LoD, who tended to associate by phone and computer, -Atlanta LoD actually WERE "tightly knit." It was no real -surprise that the Secret Service agents apprehending Urvile -at the computer-labs at Georgia Tech, would discover Prophet -with him as well. - -Urvile, a 21-year-old Georgia Tech student in polymer chemistry, -posed quite a puzzling case for law enforcement. Urvile--also known -as "Necron 99," as well as other handles, for he tended to change his -cover-alias about once a month--was both an accomplished hacker -and a fanatic simulation-gamer. - -Simulation games are an unusual hobby; but then hackers are unusual people, -and their favorite pastimes tend to be somewhat out of the ordinary. -The best-known American simulation game is probably "Dungeons & Dragons," -a multi-player parlor entertainment played with paper, maps, pencils, -statistical tables and a variety of oddly-shaped dice. Players pretend -to be heroic characters exploring a wholly-invented fantasy world. -The fantasy worlds of simulation gaming are commonly pseudo-medieval, -involving swords and sorcery--spell-casting wizards, knights in armor, -unicorns and dragons, demons and goblins. - -Urvile and his fellow gamers preferred their fantasies highly technological. -They made use of a game known as "G.U.R.P.S.," the "Generic Universal Role -Playing System," published by a company called Steve Jackson Games (SJG). - -"G.U.R.P.S." served as a framework for creating a wide variety of artificial -fantasy worlds. Steve Jackson Games published a smorgasboard of books, -full of detailed information and gaming hints, which were used to flesh-out -many different fantastic backgrounds for the basic GURPS framework. -Urvile made extensive use of two SJG books called GURPS High-Tech -and GURPS Special Ops. - -In the artificial fantasy-world of GURPS Special Ops, -players entered a modern fantasy of intrigue and international espionage. -On beginning the game, players started small and powerless, -perhaps as minor-league CIA agents or penny-ante arms dealers. -But as players persisted through a series of game sessions -(game sessions generally lasted for hours, over long, -elaborate campaigns that might be pursued for months on end) -then they would achieve new skills, new knowledge, new power. -They would acquire and hone new abilities, such as marksmanship, -karate, wiretapping, or Watergate burglary. They could also win -various kinds of imaginary booty, like Berettas, or martini shakers, -or fast cars with ejection seats and machine-guns under the headlights. - -As might be imagined from the complexity of these games, -Urvile's gaming notes were very detailed and extensive. -Urvile was a "dungeon-master," inventing scenarios -for his fellow gamers, giant simulated adventure-puzzles -for his friends to unravel. Urvile's game notes covered -dozens of pages with all sorts of exotic lunacy, all about -ninja raids on Libya and break-ins on encrypted Red Chinese supercomputers. -His notes were written on scrap-paper and kept in loose-leaf binders. - -The handiest scrap paper around Urvile's college digs were the many pounds of -BellSouth printouts and documents that he had snitched out of telco dumpsters. -His notes were written on the back of misappropriated telco property. -Worse yet, the gaming notes were chaotically interspersed with Urvile's -hand-scrawled records involving ACTUAL COMPUTER INTRUSIONS that he -had committed. - -Not only was it next to impossible to tell Urvile's fantasy game-notes -from cyberspace "reality," but Urvile himself barely made this distinction. -It's no exaggeration to say that to Urvile it was ALL a game. Urvile was -very bright, highly imaginative, and quite careless of other people's notions -of propriety. His connection to "reality" was not something to which he paid -a great deal of attention. - -Hacking was a game for Urvile. It was an amusement he was carrying out, -it was something he was doing for fun. And Urvile was an obsessive young man. -He could no more stop hacking than he could stop in the middle of -a jigsaw puzzle, or stop in the middle of reading a Stephen Donaldson -fantasy trilogy. (The name "Urvile" came from a best-selling Donaldson novel.) - -Urvile's airy, bulletproof attitude seriously annoyed his interrogators. -First of all, he didn't consider that he'd done anything wrong. -There was scarcely a shred of honest remorse in him. On the contrary, -he seemed privately convinced that his police interrogators were operating -in a demented fantasy-world all their own. Urvile was too polite -and well-behaved to say this straight-out, but his reactions were askew -and disquieting. - -For instance, there was the business about LoD's ability -to monitor phone-calls to the police and Secret Service. -Urvile agreed that this was quite possible, and posed -no big problem for LoD. In fact, he and his friends -had kicked the idea around on the "Black Ice" board, -much as they had discussed many other nifty notions, -such as building personal flame-throwers and jury-rigging -fistfulls of blasting-caps. They had hundreds of dial-up numbers -for government agencies that they'd gotten through scanning Atlanta phones, -or had pulled from raided VAX/VMS mainframe computers. - -Basically, they'd never gotten around to listening in on the cops -because the idea wasn't interesting enough to bother with. -Besides, if they'd been monitoring Secret Service phone calls, -obviously they'd never have been caught in the first place. Right? - -The Secret Service was less than satisfied with this rapier-like hacker logic. - -Then there was the issue of crashing the phone system. No problem, -Urvile admitted sunnily. Atlanta LoD could have shut down phone service -all over Atlanta any time they liked. EVEN THE 911 SERVICE? -Nothing special about that, Urvile explained patiently. -Bring the switch to its knees, with say the UNIX "makedir" bug, -and 911 goes down too as a matter of course. The 911 system -wasn't very interesting, frankly. It might be tremendously -interesting to cops (for odd reasons of their own), but as -technical challenges went, the 911 service was yawnsville. - -So of course the Atlanta Three could crash service. -They probably could have crashed service all over -BellSouth territory, if they'd worked at it for a while. -But Atlanta LoD weren't crashers. Only losers and rodents -were crashers. LoD were ELITE. - -Urvile was privately convinced that sheer technical -expertise could win him free of any kind of problem. -As far as he was concerned, elite status in the digital -underground had placed him permanently beyond the intellectual -grasp of cops and straights. Urvile had a lot to learn. - -Of the three LoD stalwarts, Prophet was in the most direct trouble. -Prophet was a UNIX programming expert who burrowed in and out -of the Internet as a matter of course. He'd started his hacking -career at around age 14, meddling with a UNIX mainframe system -at the University of North Carolina. - -Prophet himself had written the handy Legion of Doom -file "UNIX Use and Security From the Ground Up." -UNIX (pronounced "you-nicks") is a powerful, -flexible computer operating-system, for multi-user, -multi-tasking computers. In 1969, when UNIX was created -in Bell Labs, such computers were exclusive to large -corporations and universities, but today UNIX is run -on thousands of powerful home machines. UNIX was -particularly well-suited to telecommunications programming, -and had become a standard in the field. Naturally, UNIX -also became a standard for the elite hacker and phone phreak. -Lately, Prophet had not been so active as Leftist and Urvile, -but Prophet was a recidivist. In 1986, when he was eighteen, -Prophet had been convicted of "unauthorized access -to a computer network" in North Carolina. He'd been -discovered breaking into the Southern Bell Data Network, -a UNIX-based internal telco network supposedly closed to the public. -He'd gotten a typical hacker sentence: six months suspended, -120 hours community service, and three years' probation. - -After that humiliating bust, Prophet had gotten rid of most of his -tonnage of illicit phreak and hacker data, and had tried to go straight. -He was, after all, still on probation. But by the autumn of 1988, -the temptations of cyberspace had proved too much for young Prophet, -and he was shoulder-to-shoulder with Urvile and Leftist into some -of the hairiest systems around. - -In early September 1988, he'd broken into BellSouth's centralized -automation system, AIMSX or "Advanced Information Management System." -AIMSX was an internal business network for BellSouth, where telco -employees stored electronic mail, databases, memos, and calendars, -and did text processing. Since AIMSX did not have public dial-ups, -it was considered utterly invisible to the public, and was not well-secured ---it didn't even require passwords. Prophet abused an account known -as "waa1," the personal account of an unsuspecting telco employee. -Disguised as the owner of waa1, Prophet made about ten visits to AIMSX. - -Prophet did not damage or delete anything in the system. -His presence in AIMSX was harmless and almost invisible. -But he could not rest content with that. - -One particular piece of processed text on AIMSX was a telco document -known as "Bell South Standard Practice 660-225-104SV Control Office -Administration of Enhanced 911 Services for Special Services -and Major Account Centers dated March 1988." - -Prophet had not been looking for this document. It was merely one -among hundreds of similar documents with impenetrable titles. -However, having blundered over it in the course of his illicit -wanderings through AIMSX, he decided to take it with him as a trophy. -It might prove very useful in some future boasting, bragging, -and strutting session. So, some time in September 1988, -Prophet ordered the AIMSX mainframe computer to copy this document -(henceforth called simply called "the E911 Document") and to transfer -this copy to his home computer. - -No one noticed that Prophet had done this. He had "stolen" -the E911 Document in some sense, but notions of property -in cyberspace can be tricky. BellSouth noticed nothing wrong, -because BellSouth still had their original copy. They had not -been "robbed" of the document itself. Many people were supposed -to copy this document--specifically, people who worked for the -nineteen BellSouth "special services and major account centers," -scattered throughout the Southeastern United States. That was -what it was for, why it was present on a computer network -in the first place: so that it could be copied and read-- -by telco employees. But now the data had been copied -by someone who wasn't supposed to look at it. - -Prophet now had his trophy. But he further decided to store -yet another copy of the E911 Document on another person's computer. -This unwitting person was a computer enthusiast named Richard Andrews -who lived near Joliet, Illinois. Richard Andrews was a UNIX programmer -by trade, and ran a powerful UNIX board called "Jolnet," in the basement -of his house. - -Prophet, using the handle "Robert Johnson," had obtained an account -on Richard Andrews' computer. And there he stashed the E911 Document, -by storing it in his own private section of Andrews' computer. - -Why did Prophet do this? If Prophet had eliminated the E911 Document -from his own computer, and kept it hundreds of miles away, on another machine, under an -alias, then he might have been fairly safe from discovery and prosecution-- -although his sneaky action had certainly put the unsuspecting Richard Andrews -at risk. - -But, like most hackers, Prophet was a pack-rat for illicit data. -When it came to the crunch, he could not bear to part from his trophy. -When Prophet's place in Decatur, Georgia was raided in July 1989, -there was the E911 Document, a smoking gun. And there was Prophet -in the hands of the Secret Service, doing his best to "explain." - -Our story now takes us away from the Atlanta Three and their raids -of the Summer of 1989. We must leave Atlanta Three "cooperating fully" -with their numerous investigators. And all three of them did cooperate, -as their Sentencing Memorandum from the US District Court of the -Northern Division of Georgia explained--just before all three of them -were sentenced to various federal prisons in November 1990. - -We must now catch up on the other aspects of the war on the Legion of Doom. -The war on the Legion was a war on a network--in fact, a network of three -networks, which intertwined and interrelated in a complex fashion. -The Legion itself, with Atlanta LoD, and their hanger-on Fry Guy, -were the first network. The second network was Phrack magazine, -with its editors and contributors. - -The third network involved the electronic circle around a hacker -known as "Terminus." - -The war against these hacker networks was carried out by -a law enforcement network. Atlanta LoD and Fry Guy -were pursued by USSS agents and federal prosecutors in Atlanta, -Indiana, and Chicago. "Terminus" found himself pursued by USSS -and federal prosecutors from Baltimore and Chicago. And the war -against Phrack was almost entirely a Chicago operation. - -The investigation of Terminus involved a great deal of energy, -mostly from the Chicago Task Force, but it was to be the least-known -and least-publicized of the Crackdown operations. Terminus, who lived -in Maryland, was a UNIX programmer and consultant, fairly well-known -(under his given name) in the UNIX community, as an acknowledged expert -on AT&T minicomputers. Terminus idolized AT&T, especially Bellcore, -and longed for public recognition as a UNIX expert; his highest ambition -was to work for Bell Labs. - -But Terminus had odd friends and a spotted history. -Terminus had once been the subject of an admiring interview -in Phrack (Volume II, Issue 14, Phile 2--dated May 1987). -In this article, Phrack co-editor Taran King described -"Terminus" as an electronics engineer, 5'9", brown-haired, -born in 1959--at 28 years old, quite mature for a hacker. - -Terminus had once been sysop of a phreak/hack underground board -called "MetroNet," which ran on an Apple II. Later he'd replaced -"MetroNet" with an underground board called "MegaNet," -specializing in IBMs. In his younger days, Terminus had written -one of the very first and most elegant code-scanning programs -for the IBM-PC. This program had been widely distributed -in the underground. Uncounted legions of PC-owning phreaks and -hackers had used Terminus's scanner program to rip-off telco codes. -This feat had not escaped the attention of telco security; -it hardly could, since Terminus's earlier handle, "Terminal Technician," -was proudly written right on the program. - -When he became a full-time computer professional -(specializing in telecommunications programming), -he adopted the handle Terminus, meant to indicate that he -had "reached the final point of being a proficient hacker." -He'd moved up to the UNIX-based "Netsys" board on an AT&T computer, -with four phone lines and an impressive 240 megs of storage. -"Netsys" carried complete issues of Phrack, and Terminus was -quite friendly with its publishers, Taran King and Knight Lightning. - -In the early 1980s, Terminus had been a regular on Plovernet, -Pirate-80, Sherwood Forest and Shadowland, all well-known pirate boards, -all heavily frequented by the Legion of Doom. As it happened, Terminus -was never officially "in LoD," because he'd never been given the official -LoD high-sign and back-slap by Legion maven Lex Luthor. Terminus had -never physically met anyone from LoD. But that scarcely mattered much-- -the Atlanta Three themselves had never been officially vetted by Lex, either. - -As far as law enforcement was concerned, the issues were clear. -Terminus was a full-time, adult computer professional -with particular skills at AT&T software and hardware-- -but Terminus reeked of the Legion of Doom and the underground. - -On February 1, 1990--half a month after the Martin Luther King Day Crash-- -USSS agents Tim Foley from Chicago, and Jack Lewis from the Baltimore office, -accompanied by AT&T security officer Jerry Dalton, travelled to Middle Town, -Maryland. There they grilled Terminus in his home (to the stark terror of -his wife and small children), and, in their customary fashion, hauled his -computers out the door. - -The Netsys machine proved to contain a plethora of arcane UNIX software-- -proprietary source code formally owned by AT&T. Software such as: -UNIX System Five Release 3.2; UNIX SV Release 3.1; UUCP communications -software; KORN SHELL; RFS; IWB; WWB; DWB; the C++ programming language; -PMON; TOOL CHEST; QUEST; DACT, and S FIND. - -In the long-established piratical tradition of the underground, -Terminus had been trading this illicitly-copied software with -a small circle of fellow UNIX programmers. Very unwisely, -he had stored seven years of his electronic mail on his Netsys machine, -which documented all the friendly arrangements he had made with -his various colleagues. - -Terminus had not crashed the AT&T phone system on January 15. -He was, however, blithely running a not-for-profit AT&T -software-piracy ring. This was not an activity AT&T found amusing. -AT&T security officer Jerry Dalton valued this "stolen" property -at over three hundred thousand dollars. - -AT&T's entry into the tussle of free enterprise had been complicated -by the new, vague groundrules of the information economy. -Until the break-up of Ma Bell, AT&T was forbidden to sell -computer hardware or software. Ma Bell was the phone company; -Ma Bell was not allowed to use the enormous revenue from -telephone utilities, in order to finance any entry into -the computer market. - -AT&T nevertheless invented the UNIX operating system. -And somehow AT&T managed to make UNIX a minor source of income. -Weirdly, UNIX was not sold as computer software, -but actually retailed under an obscure regulatory -exemption allowing sales of surplus equipment and scrap. -Any bolder attempt to promote or retail UNIX would have -aroused angry legal opposition from computer companies. -Instead, UNIX was licensed to universities, at modest rates, -where the acids of academic freedom ate away steadily at AT&T's -proprietary rights. - -Come the breakup, AT&T recognized that UNIX was a potential gold-mine. -By now, large chunks of UNIX code had been created that were not AT&T's, -and were being sold by others. An entire rival UNIX-based operating system -had arisen in Berkeley, California (one of the world's great founts of -ideological hackerdom). Today, "hackers" commonly consider "Berkeley UNIX" -to be technically superior to AT&T's "System V UNIX," but AT&T has not -allowed mere technical elegance to intrude on the real-world business -of marketing proprietary software. AT&T has made its own code deliberately -incompatible with other folks' UNIX, and has written code that it can prove -is copyrightable, even if that code happens to be somewhat awkward--"kludgey." -AT&T UNIX user licenses are serious business agreements, replete with very -clear copyright statements and non-disclosure clauses. - -AT&T has not exactly kept the UNIX cat in the bag, -but it kept a grip on its scruff with some success. -By the rampant, explosive standards of software piracy, -AT&T UNIX source code is heavily copyrighted, well-guarded, -well-licensed. UNIX was traditionally run only on -mainframe machines, owned by large groups of suit-and-tie -professionals, rather than on bedroom machines where -people can get up to easy mischief. - -And AT&T UNIX source code is serious high-level programming. -The number of skilled UNIX programmers with any actual motive -to swipe UNIX source code is small. It's tiny, compared to -the tens of thousands prepared to rip-off, say, entertaining -PC games like "Leisure Suit Larry." - -But by 1989, the warez-d00d underground, in the persons of Terminus -and his friends, was gnawing at AT&T UNIX. And the property in question -was not sold for twenty bucks over the counter at the local branch of -Babbage's or Egghead's; this was massive, sophisticated, multi-line, -multi-author corporate code worth tens of thousands of dollars. - -It must be recognized at this point that Terminus's purported ring of UNIX -software pirates had not actually made any money from their suspected crimes. -The $300,000 dollar figure bandied about for the contents of Terminus's -computer did not mean that Terminus was in actual illicit possession -of three hundred thousand of AT&T's dollars. Terminus was shipping -software back and forth, privately, person to person, for free. -He was not making a commercial business of piracy. He hadn't -asked for money; he didn't take money. He lived quite modestly. - -AT&T employees--as well as freelance UNIX consultants, like Terminus-- -commonly worked with "proprietary" AT&T software, both in the office -and at home on their private machines. AT&T rarely sent security officers -out to comb the hard disks of its consultants. Cheap freelance UNIX -contractors were quite useful to AT&T; they didn't have health insurance -or retirement programs, much less union membership in the Communication -Workers of America. They were humble digital drudges, wandering with mop -and bucket through the Great Technological Temple of AT&T; but when the -Secret Service arrived at their homes, it seemed they were eating with -company silverware and sleeping on company sheets! Outrageously, they -behaved as if the things they worked with every day belonged to them! - -And these were no mere hacker teenagers with their hands full -of trash-paper and their noses pressed to the corporate windowpane. -These guys were UNIX wizards, not only carrying AT&T data in their -machines and their heads, but eagerly networking about it, -over machines that were far more powerful than anything previously -imagined in private hands. How do you keep people disposable, -yet assure their awestruck respect for your property? It was a dilemma. - -Much UNIX code was public-domain, available for free. Much "proprietary" -UNIX code had been extensively re-written, perhaps altered so much that it -became an entirely new product--or perhaps not. Intellectual property rights -for software developers were, and are, extraordinarily complex and confused. -And software "piracy," like the private copying of videos, is one of the most -widely practiced "crimes" in the world today. - -The USSS were not experts in UNIX or familiar with the customs of its use. -The United States Secret Service, considered as a body, did not have one single -person in it who could program in a UNIX environment--no, not even one. -The Secret Service WERE making extensive use of expert help, but the "experts" -they had chosen were AT&T and Bellcore security officials, the very victims of -the purported crimes under investigation, the very people whose interest in -AT&T's "proprietary" software was most pronounced. - -On February 6, 1990, Terminus was arrested by Agent Lewis. -Eventually, Terminus would be sent to prison for his illicit -use of a piece of AT&T software. - -The issue of pirated AT&T software would bubble along in the background -during the war on the Legion of Doom. Some half-dozen of Terminus's on-line -acquaintances, including people in Illinois, Texas and California, -were grilled by the Secret Service in connection with the illicit -copying of software. Except for Terminus, however, none were charged -with a crime. None of them shared his peculiar prominence in the -hacker underground. - -But that did not mean that these people would, or could, -stay out of trouble. The transferral of illicit data in -cyberspace is hazy and ill-defined business, with paradoxical -dangers for everyone concerned: hackers, signal carriers, -board owners, cops, prosecutors, even random passers-by. -Sometimes, well-meant attempts to avert trouble -or punish wrongdoing bring more trouble than -would simple ignorance, indifference or impropriety. - -Terminus's "Netsys" board was not a common-or-garden -bulletin board system, though it had most of the usual -functions of a board. Netsys was not a stand-alone machine, -but part of the globe-spanning "UUCP" cooperative network. -The UUCP network uses a set of Unix software programs called -"Unix-to-Unix Copy," which allows Unix systems to throw data to -one another at high speed through the public telephone network. -UUCP is a radically decentralized, not-for-profit network of UNIX computers. -There are tens of thousands of these UNIX machines. Some are small, -but many are powerful and also link to other networks. UUCP has -certain arcane links to major networks such as JANET, EasyNet, BITNET, -JUNET, VNET, DASnet, PeaceNet and FidoNet, as well as the gigantic Internet. -(The so-called "Internet" is not actually a network itself, but rather an -"internetwork" connections standard that allows several globe-spanning -computer networks to communicate with one another. Readers fascinated -by the weird and intricate tangles of modern computer networks may enjoy -John S. Quarterman's authoritative 719-page explication, The Matrix, -Digital Press, 1990.) - -A skilled user of Terminus' UNIX machine could send and receive -electronic mail from almost any major computer network in the world. -Netsys was not called a "board" per se, but rather a "node." -"Nodes" were larger, faster, and more sophisticated than mere "boards," -and for hackers, to hang out on internationally-connected "nodes" -was quite the step up from merely hanging out on local "boards." - -Terminus's Netsys node in Maryland had a number of direct -links to other, similar UUCP nodes, run by people who shared his -interests and at least something of his free-wheeling attitude. -One of these nodes was Jolnet, owned by Richard Andrews, who, -like Terminus, was an independent UNIX consultant. -Jolnet also ran UNIX, and could be contacted at high speed -by mainframe machines from all over the world. Jolnet was -quite a sophisticated piece of work, technically speaking, -but it was still run by an individual, as a private, -not-for-profit hobby. Jolnet was mostly used by other -UNIX programmers--for mail, storage, and access to networks. -Jolnet supplied access network access to about two hundred people, -as well as a local junior college. - -Among its various features and services, Jolnet also carried -Phrack magazine. - -For reasons of his own, Richard Andrews had become suspicious -of a new user called "Robert Johnson." Richard Andrews -took it upon himself to have a look at what "Robert Johnson" -was storing in Jolnet. And Andrews found the E911 Document. - -"Robert Johnson" was the Prophet from the Legion of Doom, -and the E911 Document was illicitly copied data from Prophet's -raid on the BellSouth computers. - -The E911 Document, a particularly illicit piece of digital property, -was about to resume its long, complex, and disastrous career. - -It struck Andrews as fishy that someone not a telephone employee -should have a document referring to the "Enhanced 911 System." -Besides, the document itself bore an obvious warning. - -"WARNING: NOT FOR USE OR DISCLOSURE OUTSIDE BELLSOUTH -OR ANY OF ITS SUBSIDIARIES EXCEPT UNDER WRITTEN AGREEMENT." - -These standard nondisclosure tags are often appended to all sorts -of corporate material. Telcos as a species are particularly notorious -for stamping most everything in sight as "not for use or disclosure." -Still, this particular piece of data was about the 911 System. -That sounded bad to Rich Andrews. - -Andrews was not prepared to ignore this sort of trouble. -He thought it would be wise to pass the document along -to a friend and acquaintance on the UNIX network, for consultation. -So, around September 1988, Andrews sent yet another copy of the -E911 Document electronically to an AT&T employee, one Charles Boykin, -who ran a UNIX-based node called "attctc" in Dallas, Texas. - -"Attctc" was the property of AT&T, and was run from AT&T's -Customer Technology Center in Dallas, hence the name "attctc." -"Attctc" was better-known as "Killer," the name of the machine -that the system was running on. "Killer" was a hefty, powerful, -AT&T 3B2 500 model, a multi-user, multi-tasking UNIX platform -with 32 meg of memory and a mind-boggling 3.2 Gigabytes of storage. -When Killer had first arrived in Texas, in 1985, the 3B2 had been -one of AT&T's great white hopes for going head-to-head with IBM -for the corporate computer-hardware market. "Killer" had been shipped -to the Customer Technology Center in the Dallas Infomart, essentially -a high-technology mall, and there it sat, a demonstration model. - -Charles Boykin, a veteran AT&T hardware and digital communications expert, -was a local technical backup man for the AT&T 3B2 system. As a display model -in the Infomart mall, "Killer" had little to do, and it seemed a shame -to waste the system's capacity. So Boykin ingeniously wrote some UNIX -bulletin-board software for "Killer," and plugged the machine in to the -local phone network. "Killer's" debut in late 1985 made it the first -publicly available UNIX site in the state of Texas. Anyone who wanted to -play was welcome. - -The machine immediately attracted an electronic community. -It joined the UUCP network, and offered network links -to over eighty other computer sites, all of which became dependent -on Killer for their links to the greater world of cyberspace. -And it wasn't just for the big guys; personal computer users -also stored freeware programs for the Amiga, the Apple, -the IBM and the Macintosh on Killer's vast 3,200 meg archives. -At one time, Killer had the largest library of public-domain -Macintosh software in Texas. - -Eventually, Killer attracted about 1,500 users, -all busily communicating, uploading and downloading, -getting mail, gossipping, and linking to arcane -and distant networks. - -Boykin received no pay for running Killer. He considered -it good publicity for the AT&T 3B2 system (whose sales were -somewhat less than stellar), but he also simply enjoyed -the vibrant community his skill had created. He gave away -the bulletin-board UNIX software he had written, free of charge. - -In the UNIX programming community, Charlie Boykin had the -reputation of a warm, open-hearted, level-headed kind of guy. -In 1989, a group of Texan UNIX professionals voted Boykin -"System Administrator of the Year." He was considered -a fellow you could trust for good advice. - -In September 1988, without warning, the E911 Document -came plunging into Boykin's life, forwarded by Richard Andrews. -Boykin immediately recognized that the Document was hot property. -He was not a voice-communications man, and knew little about -the ins and outs of the Baby Bells, but he certainly knew what -the 911 System was, and he was angry to see confidential data -about it in the hands of a nogoodnik. This was clearly a -matter for telco security. So, on September 21, 1988, Boykin -made yet ANOTHER copy of the E911 Document and passed this -one along to a professional acquaintance of his, one Jerome Dalton, -from AT&T Corporate Information Security. Jerry Dalton was the -very fellow who would later raid Terminus's house. - -From AT&T's security division, the E911 Document went to Bellcore. - -Bellcore (or BELL COmmunications REsearch) had once been the central -laboratory of the Bell System. Bell Labs employees had invented -the UNIX operating system. Now Bellcore was a quasi-independent, -jointly owned company that acted as the research arm for all seven -of the Baby Bell RBOCs. Bellcore was in a good position to co-ordinate -security technology and consultation for the RBOCs, and the gentleman in -charge of this effort was Henry M. Kluepfel, a veteran of the Bell System -who had worked there for twenty-four years. - -On October 13, 1988, Dalton passed the E911 Document to Henry Kluepfel. -Kluepfel, a veteran expert witness in telecommunications fraud and -computer-fraud cases, had certainly seen worse trouble than this. -He recognized the document for what it was: a trophy from a hacker break-in. - -However, whatever harm had been done in the intrusion was presumably old news. -At this point there seemed little to be done. Kluepfel made a careful note -of the circumstances and shelved the problem for the time being. - -Whole months passed. - -February 1989 arrived. The Atlanta Three were living it up -in Bell South's switches, and had not yet met their comeuppance. -The Legion was thriving. So was Phrack magazine. -A good six months had passed since Prophet's AIMSX break-in. -Prophet, as hackers will, grew weary of sitting on his laurels. -"Knight Lightning" and "Taran King," the editors of Phrack, -were always begging Prophet for material they could publish. -Prophet decided that the heat must be off by this time, -and that he could safely brag, boast, and strut. - -So he sent a copy of the E911 Document--yet another one-- -from Rich Andrews' Jolnet machine to Knight Lightning's -BITnet account at the University of Missouri. -Let's review the fate of the document so far. - -0. The original E911 Document. This in the AIMSX system -on a mainframe computer in Atlanta, available to hundreds of people, -but all of them, presumably, BellSouth employees. An unknown number -of them may have their own copies of this document, but they are all -professionals and all trusted by the phone company. - -1. Prophet's illicit copy, at home on his own computer in Decatur, Georgia. - -2. Prophet's back-up copy, stored on Rich Andrew's Jolnet machine - in the basement of Rich Andrews' house near Joliet Illinois. - -3. Charles Boykin's copy on "Killer" in Dallas, Texas, - sent by Rich Andrews from Joliet. - -4. Jerry Dalton's copy at AT&T Corporate Information Security in New Jersey, - sent from Charles Boykin in Dallas. - -5. Henry Kluepfel's copy at Bellcore security headquarters in New Jersey, - sent by Dalton. -6. Knight Lightning's copy, sent by Prophet from Rich Andrews' machine, - and now in Columbia, Missouri. - -We can see that the "security" situation of this proprietary document, -once dug out of AIMSX, swiftly became bizarre. Without any money -changing hands, without any particular special effort, this data -had been reproduced at least six times and had spread itself all over -the continent. By far the worst, however, was yet to come. - -In February 1989, Prophet and Knight Lightning bargained electronically -over the fate of this trophy. Prophet wanted to boast, but, at the same time, -scarcely wanted to be caught. - -For his part, Knight Lightning was eager to publish as much of the document -as he could manage. Knight Lightning was a fledgling political-science major -with a particular interest in freedom-of-information issues. He would gladly -publish most anything that would reflect glory on the prowess of the -underground and embarrass the telcos. However, Knight Lightning himself -had contacts in telco security, and sometimes consulted them on material -he'd received that might be too dicey for publication. - -Prophet and Knight Lightning decided to edit the E911 Document -so as to delete most of its identifying traits. First of all, -its large "NOT FOR USE OR DISCLOSURE" warning had to go. -Then there were other matters. For instance, it listed -the office telephone numbers of several BellSouth 911 -specialists in Florida. If these phone numbers were -published in Phrack, the BellSouth employees involved -would very likely be hassled by phone phreaks, -which would anger BellSouth no end, and pose a -definite operational hazard for both Prophet and Phrack. - -So Knight Lightning cut the Document almost in half, -removing the phone numbers and some of the touchier -and more specific information. He passed it back -electronically to Prophet; Prophet was still nervous, -so Knight Lightning cut a bit more. They finally agreed -that it was ready to go, and that it would be published -in Phrack under the pseudonym, "The Eavesdropper." - -And this was done on February 25, 1989. - -The twenty-fourth issue of Phrack featured a chatty interview -with co-ed phone-phreak "Chanda Leir," three articles on BITNET -and its links to other computer networks, an article on 800 and 900 -numbers by "Unknown User," "VaxCat's" article on telco basics -(slyly entitled "Lifting Ma Bell's Veil of Secrecy,)" and -the usual "Phrack World News." - -The News section, with painful irony, featured an extended account -of the sentencing of "Shadowhawk," an eighteen-year-old Chicago hacker -who had just been put in federal prison by William J. Cook himself. - -And then there were the two articles by "The Eavesdropper." -The first was the edited E911 Document, now titled -"Control Office Administration Of Enhanced 911 Services -for Special Services and Major Account Centers." -Eavesdropper's second article was a glossary of terms -explaining the blizzard of telco acronyms and buzzwords -in the E911 Document. - -The hapless document was now distributed, in the usual Phrack routine, -to a good one hundred and fifty sites. Not a hundred and fifty PEOPLE, -mind you--a hundred and fifty SITES, some of these sites linked to UNIX -nodes or bulletin board systems, which themselves had readerships of tens, -dozens, even hundreds of people. - -This was February 1989. Nothing happened immediately. -Summer came, and the Atlanta crew were raided by the Secret Service. -Fry Guy was apprehended. Still nothing whatever happened to Phrack. -Six more issues of Phrack came out, 30 in all, more or less on -a monthly schedule. Knight Lightning and co-editor Taran King -went untouched. - -Phrack tended to duck and cover whenever the heat came down. -During the summer busts of 1987--(hacker busts tended to cluster in summer, -perhaps because hackers were easier to find at home than in college)-- -Phrack had ceased publication for several months, and laid low. -Several LoD hangers-on had been arrested, but nothing had happened -to the Phrack crew, the premiere gossips of the underground. -In 1988, Phrack had been taken over by a new editor, -"Crimson Death," a raucous youngster with a taste for anarchy files. -1989, however, looked like a bounty year for the underground. -Knight Lightning and his co-editor Taran King took up the reins again, -and Phrack flourished throughout 1989. Atlanta LoD went down hard in -the summer of 1989, but Phrack rolled merrily on. Prophet's E911 Document -seemed unlikely to cause Phrack any trouble. By January 1990, -it had been available in Phrack for almost a year. Kluepfel and Dalton, -officers of Bellcore and AT&T security, had possessed the document -for sixteen months--in fact, they'd had it even before Knight Lightning -himself, and had done nothing in particular to stop its distribution. -They hadn't even told Rich Andrews or Charles Boykin to erase the copies -from their UNIX nodes, Jolnet and Killer. - -But then came the monster Martin Luther King Day Crash of January 15, 1990. - -A flat three days later, on January 18, four agents showed up -at Knight Lightning's fraternity house. One was Timothy Foley, -the second Barbara Golden, both of them Secret Service agents -from the Chicago office. Also along was a University of Missouri -security officer, and Reed Newlin, a security man from Southwestern Bell, -the RBOC having jurisdiction over Missouri. - -Foley accused Knight Lightning of causing the nationwide crash -of the phone system. - -Knight Lightning was aghast at this allegation. On the face of it, -the suspicion was not entirely implausible--though Knight Lightning -knew that he himself hadn't done it. Plenty of hot-dog hackers -had bragged that they could crash the phone system, however. -"Shadowhawk," for instance, the Chicago hacker whom William Cook -had recently put in jail, had several times boasted on boards -that he could "shut down AT&T's public switched network." - -And now this event, or something that looked just like it, -had actually taken place. The Crash had lit a fire under -the Chicago Task Force. And the former fence-sitters at -Bellcore and AT&T were now ready to roll. The consensus -among telco security--already horrified by the skill of -the BellSouth intruders --was that the digital underground -was out of hand. LoD and Phrack must go. And in publishing -Prophet's E911 Document, Phrack had provided law enforcement -with what appeared to be a powerful legal weapon. - -Foley confronted Knight Lightning about the E911 Document. - -Knight Lightning was cowed. He immediately began "cooperating fully" -in the usual tradition of the digital underground. - -He gave Foley a complete run of Phrack, printed out in a set -of three-ring binders. He handed over his electronic mailing list -of Phrack subscribers. Knight Lightning was grilled for four hours -by Foley and his cohorts. Knight Lightning admitted that Prophet -had passed him the E911 Document, and he admitted that he had known -it was stolen booty from a hacker raid on a telephone company. -Knight Lightning signed a statement to this effect, and agreed, -in writing, to cooperate with investigators. - -Next day--January 19, 1990, a Friday --the Secret Service returned -with a search warrant, and thoroughly searched Knight Lightning's -upstairs room in the fraternity house. They took all his floppy disks, -though, interestingly, they left Knight Lightning in possession -of both his computer and his modem. (The computer had no hard disk, -and in Foley's judgement was not a store of evidence.) But this was a -very minor bright spot among Knight Lightning's rapidly multiplying troubles. -By this time, Knight Lightning was in plenty of hot water, not only with -federal police, prosecutors, telco investigators, and university security, -but with the elders of his own campus fraternity, who were outraged -to think that they had been unwittingly harboring a federal computer-criminal. - -On Monday, Knight Lightning was summoned to Chicago, where he was -further grilled by Foley and USSS veteran agent Barbara Golden, this time -with an attorney present. And on Tuesday, he was formally indicted -by a federal grand jury. - -The trial of Knight Lightning, which occurred on July 24-27, 1990, -was the crucial show-trial of the Hacker Crackdown. We will examine -the trial at some length in Part Four of this book. - -In the meantime, we must continue our dogged pursuit of the E911 Document. - -It must have been clear by January 1990 that the E911 Document, -in the form Phrack had published it back in February 1989, -had gone off at the speed of light in at least a hundred -and fifty different directions. To attempt to put this -electronic genie back in the bottle was flatly impossible. - -And yet, the E911 Document was STILL stolen property, -formally and legally speaking. Any electronic transference -of this document, by anyone unauthorized to have it, -could be interpreted as an act of wire fraud. Interstate -transfer of stolen property, including electronic property, -was a federal crime. - -The Chicago Computer Fraud and Abuse Task Force had been assured -that the E911 Document was worth a hefty sum of money. In fact, -they had a precise estimate of its worth from BellSouth security personnel: -$79,449. A sum of this scale seemed to warrant vigorous prosecution. -Even if the damage could not be undone, at least this large sum -offered a good legal pretext for stern punishment of the thieves. -It seemed likely to impress judges and juries. And it could be used -in court to mop up the Legion of Doom. - -The Atlanta crowd was already in the bag, by the time -the Chicago Task Force had gotten around to Phrack. -But the Legion was a hydra-headed thing. In late 89, -a brand-new Legion of Doom board, "Phoenix Project," -had gone up in Austin, Texas. Phoenix Project was sysoped -by no less a man than the Mentor himself, ably assisted by -University of Texas student and hardened Doomster "Erik Bloodaxe." - -As we have seen from his Phrack manifesto, the Mentor was a hacker -zealot who regarded computer intrusion as something close to a moral duty. -Phoenix Project was an ambitious effort, intended to revive the digital -underground to what Mentor considered the full flower of the early 80s. -The Phoenix board would also boldly bring elite hackers face-to-face -with the telco "opposition." On "Phoenix," America's cleverest hackers -would supposedly shame the telco squareheads out of their stick-in-the-mud -attitudes, and perhaps convince them that the Legion of Doom elite were really -an all-right crew. The premiere of "Phoenix Project" was heavily trumpeted -by Phrack,and "Phoenix Project" carried a complete run of Phrack issues, -including the E911 Document as Phrack had published it. - -Phoenix Project was only one of many--possibly hundreds--of nodes and boards -all over America that were in guilty possession of the E911 Document. -But Phoenix was an outright, unashamed Legion of Doom board. -Under Mentor's guidance, it was flaunting itself in the face -of telco security personnel. Worse yet, it was actively trying -to WIN THEM OVER as sympathizers for the digital underground elite. -"Phoenix" had no cards or codes on it. Its hacker elite considered -Phoenix at least technically legal. But Phoenix was a corrupting influence, -where hacker anarchy was eating away like digital acid at the underbelly -of corporate propriety. - -The Chicago Computer Fraud and Abuse Task Force now prepared -to descend upon Austin, Texas. - -Oddly, not one but TWO trails of the Task Force's investigation led -toward Austin. The city of Austin, like Atlanta, had made itself -a bulwark of the Sunbelt's Information Age, with a strong university -research presence, and a number of cutting-edge electronics companies, -including Motorola, Dell, CompuAdd, IBM, Sematech and MCC. - -Where computing machinery went, hackers generally followed. -Austin boasted not only "Phoenix Project," currently LoD's -most flagrant underground board, but a number of UNIX nodes. - -One of these nodes was "Elephant," run by a UNIX consultant -named Robert Izenberg. Izenberg, in search of a relaxed Southern -lifestyle and a lowered cost-of-living, had recently migrated -to Austin from New Jersey. In New Jersey, Izenberg had worked -for an independent contracting company, programming UNIX code for -AT&T itself. "Terminus" had been a frequent user on Izenberg's -privately owned Elephant node. - -Having interviewed Terminus and examined the records on Netsys, -the Chicago Task Force were now convinced that they had discovered -an underground gang of UNIX software pirates, who were demonstrably -guilty of interstate trafficking in illicitly copied AT&T source code. -Izenberg was swept into the dragnet around Terminus, the self-proclaimed -ultimate UNIX hacker. - -Izenberg, in Austin, had settled down into a UNIX job -with a Texan branch of IBM. Izenberg was no longer -working as a contractor for AT&T, but he had friends -in New Jersey, and he still logged on to AT&T UNIX -computers back in New Jersey, more or less whenever -it pleased him. Izenberg's activities appeared highly -suspicious to the Task Force. Izenberg might well be -breaking into AT&T computers, swiping AT&T software, -and passing it to Terminus and other possible confederates, -through the UNIX node network. And this data was worth, -not merely $79,499, but hundreds of thousands of dollars! - -On February 21, 1990, Robert Izenberg arrived home -from work at IBM to find that all the computers -had mysteriously vanished from his Austin apartment. -Naturally he assumed that he had been robbed. -His "Elephant" node, his other machines, his notebooks, -his disks, his tapes, all gone! However, nothing much -else seemed disturbed--the place had not been ransacked. -The puzzle becaming much stranger some five minutes later. -Austin U. S. Secret Service Agent Al Soliz, accompanied by -University of Texas campus-security officer Larry Coutorie -and the ubiquitous Tim Foley, made their appearance at Izenberg's door. -They were in plain clothes: slacks, polo shirts. They came in, -and Tim Foley accused Izenberg of belonging to the Legion of Doom. - -Izenberg told them that he had never heard of the "Legion of Doom." -And what about a certain stolen E911 Document, that posed a direct -threat to the police emergency lines? Izenberg claimed that he'd -never heard of that, either. - -His interrogators found this difficult to believe. -Didn't he know Terminus? - -Who? - -They gave him Terminus's real name. Oh yes, said Izenberg. -He knew THAT guy all right--he was leading discussions -on the Internet about AT&T computers, especially the AT&T 3B2. - -AT&T had thrust this machine into the marketplace, -but, like many of AT&T's ambitious attempts to enter -the computing arena, the 3B2 project had something less -than a glittering success. Izenberg himself had been -a contractor for the division of AT&T that supported the 3B2. -The entire division had been shut down. - -Nowadays, the cheapest and quickest way to get help with this -fractious piece of machinery was to join one of Terminus's -discussion groups on the Internet, where friendly and knowledgeable -hackers would help you for free. Naturally the remarks within this -group were less than flattering about the Death Star. . .was -THAT the problem? - -Foley told Izenberg that Terminus had been acquiring hot software -through his, Izenberg's, machine. - -Izenberg shrugged this off. A good eight megabytes of data flowed -through his UUCP site every day. UUCP nodes spewed data like fire hoses. -Elephant had been directly linked to Netsys--not surprising, since Terminus -was a 3B2 expert and Izenberg had been a 3B2 contractor. -Izenberg was also linked to "attctc" and the University of Texas. -Terminus was a well-known UNIX expert, and might have been up to -all manner of hijinks on Elephant. Nothing Izenberg could do about that. -That was physically impossible. Needle in a haystack. - -In a four-hour grilling, Foley urged Izenberg to come clean -and admit that he was in conspiracy with Terminus, -and a member of the Legion of Doom. - -Izenberg denied this. He was no weirdo teenage hacker-- -he was thirty-two years old, and didn't even have a "handle." -Izenberg was a former TV technician and electronics specialist -who had drifted into UNIX consulting as a full-grown adult. -Izenberg had never met Terminus, physically. He'd once bought -a cheap high-speed modem from him, though. - -Foley told him that this modem (a Telenet T2500 which ran at 19.2 kilobaud, -and which had just gone out Izenberg's door in Secret Service custody) -was likely hot property. Izenberg was taken aback to hear this; but then -again, most of Izenberg's equipment, like that of most freelance professionals -in the industry, was discounted, passed hand-to-hand through various kinds -of barter and gray-market. There was no proof that the modem was stolen, -and even if it were, Izenberg hardly saw how that gave them the right -to take every electronic item in his house. - -Still, if the United States Secret Service figured they needed -his computer for national security reasons--or whatever-- -then Izenberg would not kick. He figured he would somehow -make the sacrifice of his twenty thousand dollars' worth -of professional equipment, in the spirit of full cooperation -and good citizenship. - -Robert Izenberg was not arrested. Izenberg was not charged with any crime. -His UUCP node--full of some 140 megabytes of the files, mail, and data -of himself and his dozen or so entirely innocent users--went out the door -as "evidence." Along with the disks and tapes, Izenberg had lost about -800 megabytes of data. - -Six months would pass before Izenberg decided to phone the Secret Service -and ask how the case was going. That was the first time that Robert Izenberg -would ever hear the name of William Cook. As of January 1992, a full -two years after the seizure, Izenberg, still not charged with any crime, -would be struggling through the morass of the courts, in hope of recovering -his thousands of dollars' worth of seized equipment. - -In the meantime, the Izenberg case received absolutely no press coverage. -The Secret Service had walked into an Austin home, removed a UNIX bulletin- -board system, and met with no operational difficulties whatsoever. - -Except that word of a crackdown had percolated through the Legion of Doom. -"The Mentor" voluntarily shut down "The Phoenix Project." It seemed a pity, -especially as telco security employees had, in fact, shown up on Phoenix, -just as he had hoped--along with the usual motley crowd of LoD heavies, -hangers-on, phreaks, hackers and wannabes. There was "Sandy" Sandquist from -US SPRINT security, and some guy named Henry Kluepfel, from Bellcore itself! -Kluepfel had been trading friendly banter with hackers on Phoenix since -January 30th (two weeks after the Martin Luther King Day Crash). -The presence of such a stellar telco official seemed quite the coup -for Phoenix Project. - -Still, Mentor could judge the climate. Atlanta in ruins, -Phrack in deep trouble, something weird going on with UNIX nodes-- -discretion was advisable. Phoenix Project went off-line. - -Kluepfel, of course, had been monitoring this LoD bulletin -board for his own purposes--and those of the Chicago unit. -As far back as June 1987, Kluepfel had logged on to a Texas -underground board called "Phreak Klass 2600." There he'd -discovered an Chicago youngster named "Shadowhawk," -strutting and boasting about rifling AT&T computer files, -and bragging of his ambitions to riddle AT&T's Bellcore -computers with trojan horse programs. Kluepfel had passed -the news to Cook in Chicago, Shadowhawk's computers -had gone out the door in Secret Service custody, -and Shadowhawk himself had gone to jail. - -Now it was Phoenix Project's turn. Phoenix Project postured -about "legality" and "merely intellectual interest," but it reeked -of the underground. It had Phrack on it. It had the E911 Document. -It had a lot of dicey talk about breaking into systems, including some -bold and reckless stuff about a supposed "decryption service" that Mentor -and friends were planning to run, to help crack encrypted passwords off -of hacked systems. - -Mentor was an adult. There was a bulletin board at his place of work, -as well. Kleupfel logged onto this board, too, and discovered it to be -called "Illuminati." It was run by some company called Steve Jackson Games. - -On March 1, 1990, the Austin crackdown went into high gear. - -On the morning of March 1--a Thursday--21-year-old University of Texas -student "Erik Bloodaxe," co-sysop of Phoenix Project and an avowed member -of the Legion of Doom, was wakened by a police revolver levelled at his head. - -Bloodaxe watched, jittery, as Secret Service agents -appropriated his 300 baud terminal and, rifling his files, -discovered his treasured source-code for Robert Morris's -notorious Internet Worm. But Bloodaxe, a wily operator, -had suspected that something of the like might be coming. -All his best equipment had been hidden away elsewhere. -The raiders took everything electronic, however, -including his telephone. They were stymied by his -hefty arcade-style Pac-Man game, and left it in place, -as it was simply too heavy to move. - -Bloodaxe was not arrested. He was not charged with any crime. -A good two years later, the police still had what they had -taken from him, however. - -The Mentor was less wary. The dawn raid rousted him and his wife -from bed in their underwear, and six Secret Service agents, -accompanied by an Austin policeman and Henry Kluepfel himself, -made a rich haul. Off went the works, into the agents' white -Chevrolet minivan: an IBM PC-AT clone with 4 meg of RAM and -a 120-meg hard disk; a Hewlett-Packard LaserJet II printer; -a completely legitimate and highly expensive SCO-Xenix 286 -operating system; Pagemaker disks and documentation; -and the Microsoft Word word-processing program. Mentor's wife -had her incomplete academic thesis stored on the hard-disk; -that went, too, and so did the couple's telephone. As of two years later, -all this property remained in police custody. - -Mentor remained under guard in his apartment as agents prepared -to raid Steve Jackson Games. The fact that this was a business -headquarters and not a private residence did not deter the agents. -It was still very early; no one was at work yet. The agents prepared -to break down the door, but Mentor, eavesdropping on the Secret Service -walkie-talkie traffic, begged them not to do it, and offered his key -to the building. - -The exact details of the next events are unclear. The agents -would not let anyone else into the building. Their search warrant, -when produced, was unsigned. Apparently they breakfasted from the local -"Whataburger," as the litter from hamburgers was later found inside. -They also extensively sampled a bag of jellybeans kept by an SJG employee. -Someone tore a "Dukakis for President" sticker from the wall. - -SJG employees, diligently showing up for the day's work, were met -at the door and briefly questioned by U.S. Secret Service agents. -The employees watched in astonishment as agents wielding crowbars -and screwdrivers emerged with captive machines. They attacked -outdoor storage units with boltcutters. The agents wore -blue nylon windbreakers with "SECRET SERVICE" stencilled -across the back, with running-shoes and jeans. - -Jackson's company lost three computers, several hard-disks, -hundred of floppy disks, two monitors, three modems, -a laser printer, various powercords, cables, and adapters -(and, oddly, a small bag of screws, bolts and nuts). -The seizure of Illuminati BBS deprived SJG of all the programs, -text files, and private e-mail on the board. The loss of two other -SJG computers was a severe blow as well, since it caused the loss -of electronically stored contracts, financial projections, -address directories, mailing lists, personnel files, -business correspondence, and, not least, the drafts -of forthcoming games and gaming books. - -No one at Steve Jackson Games was arrested. No one was accused -of any crime. No charges were filed. Everything appropriated -was officially kept as "evidence" of crimes never specified. - -After the Phrack show-trial, the Steve Jackson Games scandal -was the most bizarre and aggravating incident of the Hacker -Crackdown of 1990. This raid by the Chicago Task Force -on a science-fiction gaming publisher was to rouse a -swarming host of civil liberties issues, and gave rise -to an enduring controversy that was still re-complicating itself, -and growing in the scope of its implications, a full two years later. - -The pursuit of the E911 Document stopped with the Steve Jackson Games raid. -As we have seen, there were hundreds, perhaps thousands of computer users -in America with the E911 Document in their possession. Theoretically, -Chicago had a perfect legal right to raid any of these people, -and could have legally seized the machines of anybody who subscribed to Phrack. -However, there was no copy of the E911 Document on Jackson's Illuminati board. -And there the Chicago raiders stopped dead; they have not raided anyone since. - -It might be assumed that Rich Andrews and Charlie Boykin, who had brought -the E911 Document to the attention of telco security, might be spared -any official suspicion. But as we have seen, the willingness to -"cooperate fully" offers little, if any, assurance against federal -anti-hacker prosecution. - -Richard Andrews found himself in deep trouble, thanks to the E911 Document. -Andrews lived in Illinois, the native stomping grounds of the Chicago -Task Force. On February 3 and 6, both his home and his place of work -were raided by USSS. His machines went out the door, too, and he was -grilled at length (though not arrested). Andrews proved to be in -purportedly guilty possession of: UNIX SVR 3.2; UNIX SVR 3.1; UUCP; -PMON; WWB; IWB; DWB; NROFF; KORN SHELL '88; C++; and QUEST, -among other items. Andrews had received this proprietary code-- -which AT&T officially valued at well over $250,000--through the -UNIX network, much of it supplied to him as a personal favor by Terminus. -Perhaps worse yet, Andrews admitted to returning the favor, by passing -Terminus a copy of AT&T proprietary STARLAN source code. - -Even Charles Boykin, himself an AT&T employee, entered some very hot water. -By 1990, he'd almost forgotten about the E911 problem he'd reported in -September 88; in fact, since that date, he'd passed two more security alerts -to Jerry Dalton, concerning matters that Boykin considered far worse than -the E911 Document. - -But by 1990, year of the crackdown, AT&T Corporate Information Security -was fed up with "Killer." This machine offered no direct income to AT&T, -and was providing aid and comfort to a cloud of suspicious yokels -from outside the company, some of them actively malicious toward AT&T, -its property, and its corporate interests. Whatever goodwill and publicity -had been won among Killer's 1,500 devoted users was considered no longer -worth the security risk. On February 20, 1990, Jerry Dalton arrived in -Dallas and simply unplugged the phone jacks, to the puzzled alarm -of Killer's many Texan users. Killer went permanently off-line, -with the loss of vast archives of programs and huge quantities -of electronic mail; it was never restored to service. AT&T showed -no particular regard for the "property" of these 1,500 people. -Whatever "property" the users had been storing on AT&T's computer -simply vanished completely. - -Boykin, who had himself reported the E911 problem, -now found himself under a cloud of suspicion. In a weird -private-security replay of the Secret Service seizures, -Boykin's own home was visited by AT&T Security and his -own machines were carried out the door. - -However, there were marked special features in the Boykin case. -Boykin's disks and his personal computers were swiftly examined -by his corporate employers and returned politely in just two days-- -(unlike Secret Service seizures, which commonly take months or years). -Boykin was not charged with any crime or wrongdoing, and he kept his job -with AT&T (though he did retire from AT&T in September 1991, -at the age of 52). - -It's interesting to note that the US Secret Service somehow failed -to seize Boykin's "Killer" node and carry AT&T's own computer out the door. -Nor did they raid Boykin's home. They seemed perfectly willing to take the -word of AT&T Security that AT&T's employee, and AT&T's "Killer" node, -were free of hacker contraband and on the up-and-up. - -It's digital water-under-the-bridge at this point, as Killer's -3,200 megabytes of Texan electronic community were erased in 1990, -and "Killer" itself was shipped out of the state. - -But the experiences of Andrews and Boykin, and the users of their systems, -remained side issues. They did not begin to assume the social, political, -and legal importance that gathered, slowly but inexorably, around the issue -of the raid on Steve Jackson Games. - -# - -We must now turn our attention to Steve Jackson Games itself, -and explain what SJG was, what it really did, and how it had -managed to attract this particularly odd and virulent kind of trouble. -The reader may recall that this is not the first but the second time -that the company has appeared in this narrative; a Steve Jackson game -called GURPS was a favorite pastime of Atlanta hacker Urvile, -and Urvile's science-fictional gaming notes had been mixed up -promiscuously with notes about his actual computer intrusions. - -First, Steve Jackson Games, Inc., was NOT a publisher of "computer games." -SJG published "simulation games," parlor games that were played on paper, -with pencils, and dice, and printed guidebooks full of rules and -statistics tables. There were no computers involved in the games themselves. -When you bought a Steve Jackson Game, you did not receive any software disks. -What you got was a plastic bag with some cardboard game tokens, -maybe a few maps or a deck of cards. Most of their products were books. - -However, computers WERE deeply involved in the Steve Jackson Games business. -Like almost all modern publishers, Steve Jackson and his fifteen employees -used computers to write text, to keep accounts, and to run the business -generally. They also used a computer to run their official bulletin board -system for Steve Jackson Games, a board called Illuminati. On Illuminati, -simulation gamers who happened to own computers and modems could associate, -trade mail, debate the theory and practice of gaming, and keep up with the -company's news and its product announcements. - -Illuminati was a modestly popular board, run on a small computer -with limited storage, only one phone-line, and no ties to large-scale -computer networks. It did, however, have hundreds of users, -many of them dedicated gamers willing to call from out-of-state. - -Illuminati was NOT an "underground" board. It did not feature hints -on computer intrusion, or "anarchy files," or illicitly posted -credit card numbers, or long-distance access codes. -Some of Illuminati's users, however, were members of the Legion of Doom. -And so was one of Steve Jackson's senior employees--the Mentor. -The Mentor wrote for Phrack, and also ran an underground board, -Phoenix Project--but the Mentor was not a computer professional. -The Mentor was the managing editor of Steve Jackson Games and -a professional game designer by trade. These LoD members did not -use Illuminati to help their HACKING activities. They used it to -help their GAME-PLAYING activities--and they were even more dedicated -to simulation gaming than they were to hacking. - -"Illuminati" got its name from a card-game that Steve Jackson himself, -the company's founder and sole owner, had invented. This multi-player -card-game was one of Mr Jackson's best-known, most successful, -most technically innovative products. "Illuminati" was a game -of paranoiac conspiracy in which various antisocial cults warred -covertly to dominate the world. "Illuminati" was hilarious, -and great fun to play, involving flying saucers, the CIA, the KGB, -the phone companies, the Ku Klux Klan, the South American Nazis, -the cocaine cartels, the Boy Scouts, and dozens of other splinter groups -from the twisted depths of Mr. Jackson's professionally fervid imagination. -For the uninitiated, any public discussion of the "Illuminati" card-game -sounded, by turns, utterly menacing or completely insane. - -And then there was SJG's "Car Wars," in which souped-up armored hot-rods -with rocket-launchers and heavy machine-guns did battle on the American -highways of the future. The lively Car Wars discussion on the Illuminati -board featured many meticulous, painstaking discussions of the effects -of grenades, land-mines, flamethrowers and napalm. It sounded like -hacker anarchy files run amuck. - -Mr Jackson and his co-workers earned their daily bread by supplying people -with make-believe adventures and weird ideas. The more far-out, the better. - -Simulation gaming is an unusual pastime, but gamers have not -generally had to beg the permission of the Secret Service to exist. -Wargames and role-playing adventures are an old and honored pastime, -much favored by professional military strategists. Once little-known, -these games are now played by hundreds of thousands of enthusiasts -throughout North America, Europe and Japan. Gaming-books, once restricted -to hobby outlets, now commonly appear in chain-stores like B. Dalton's -and Waldenbooks, and sell vigorously. - -Steve Jackson Games, Inc., of Austin, Texas, was a games company -of the middle rank. In 1989, SJG grossed about a million dollars. -Jackson himself had a good reputation in his industry as a talented -and innovative designer of rather unconventional games, but his company -was something less than a titan of the field--certainly not like the -multimillion-dollar TSR Inc., or Britain's gigantic "Games Workshop." -SJG's Austin headquarters was a modest two-story brick office-suite, -cluttered with phones, photocopiers, fax machines and computers. -It bustled with semi-organized activity and was littered with -glossy promotional brochures and dog-eared science-fiction novels. -Attached to the offices was a large tin-roofed warehouse piled twenty feet -high with cardboard boxes of games and books. Despite the weird imaginings -that went on within it, the SJG headquarters was quite a quotidian, -everyday sort of place. It looked like what it was: a publishers' digs. - -Both "Car Wars" and "Illuminati" were well-known, popular games. -But the mainstay of the Jackson organization was their Generic Universal -Role-Playing System, "G.U.R.P.S." The GURPS system was considered solid -and well-designed, an asset for players. But perhaps the most popular -feature of the GURPS system was that it allowed gaming-masters to design -scenarios that closely resembled well-known books, movies, and other works -of fantasy. Jackson had licensed and adapted works from many science fiction -and fantasy authors. There was GURPS Conan, GURPS Riverworld, -GURPS Horseclans, GURPS Witch World, names eminently familiar -to science-fiction readers. And there was GURPS Special Ops, -from the world of espionage fantasy and unconventional warfare. - -And then there was GURPS Cyberpunk. - -"Cyberpunk" was a term given to certain science fiction writers -who had entered the genre in the 1980s. "Cyberpunk," as the label implies, -had two general distinguishing features. First, its writers had a compelling -interest in information technology, an interest closely akin -to science fiction's earlier fascination with space travel. -And second, these writers were "punks," with all the -distinguishing features that that implies: Bohemian artiness, -youth run wild, an air of deliberate rebellion, funny clothes and hair, -odd politics, a fondness for abrasive rock and roll; in a word, trouble. - -The "cyberpunk" SF writers were a small group of mostly college-educated -white middle-class litterateurs, scattered through the US and Canada. -Only one, Rudy Rucker, a professor of computer science in Silicon Valley, -could rank with even the humblest computer hacker. But, except for -Professor Rucker, the "cyberpunk" authors were not programmers -or hardware experts; they considered themselves artists -(as, indeed, did Professor Rucker). However, these writers -all owned computers, and took an intense and public interest -in the social ramifications of the information industry. - -The cyberpunks had a strong following among the global generation -that had grown up in a world of computers, multinational networks, -and cable television. Their outlook was considered somewhat morbid, -cynical, and dark, but then again, so was the outlook of their -generational peers. As that generation matured and increased -in strength and influence, so did the cyberpunks. -As science-fiction writers went, they were doing -fairly well for themselves. By the late 1980s, -their work had attracted attention from gaming companies, -including Steve Jackson Games, which was planning a cyberpunk -simulation for the flourishing GURPS gaming-system. - -The time seemed ripe for such a product, which had already been proven -in the marketplace. The first games- company out of the gate, -with a product boldly called "Cyberpunk" in defiance of possible -infringement-of-copyright suits, had been an upstart group called -R. Talsorian. Talsorian's Cyberpunk was a fairly decent game, -but the mechanics of the simulation system left a lot to be desired. -Commercially, however, the game did very well. - -The next cyberpunk game had been the even more successful Shadowrun -by FASA Corporation. The mechanics of this game were fine, but the -scenario was rendered moronic by sappy fantasy elements like elves, -trolls, wizards, and dragons--all highly ideologically-incorrect, -according to the hard-edged, high-tech standards of cyberpunk science fiction. - -Other game designers were champing at the bit. Prominent among them -was the Mentor, a gentleman who, like most of his friends in the -Legion of Doom, was quite the cyberpunk devotee. Mentor reasoned -that the time had come for a REAL cyberpunk gaming-book--one that the -princes of computer-mischief in the Legion of Doom could play without -laughing themselves sick. This book, GURPS Cyberpunk, would reek -of culturally on-line authenticity. - -Mentor was particularly well-qualified for this task. -Naturally, he knew far more about computer-intrusion -and digital skullduggery than any previously published -cyberpunk author. Not only that, but he was good at his work. -A vivid imagination, combined with an instinctive feeling -for the working of systems and, especially, the loopholes -within them, are excellent qualities for a professional game designer. - -By March 1st, GURPS Cyberpunk was almost complete, ready to print and ship. -Steve Jackson expected vigorous sales for this item, which, he hoped, -would keep the company financially afloat for several months. -GURPS Cyberpunk, like the other GURPS "modules," was not a "game" -like a Monopoly set, but a BOOK: a bound paperback book the size -of a glossy magazine, with a slick color cover, and pages full of text, -illustrations, tables and footnotes. It was advertised as a game, -and was used as an aid to game-playing, but it was a book, -with an ISBN number, published in Texas, copyrighted, -and sold in bookstores. - -And now, that book, stored on a computer, had gone out the door -in the custody of the Secret Service. - -The day after the raid, Steve Jackson visited the local Secret Service -headquarters with a lawyer in tow. There he confronted Tim Foley -(still in Austin at that time) and demanded his book back. But there -was trouble. GURPS Cyberpunk, alleged a Secret Service agent to astonished -businessman Steve Jackson, was "a manual for computer crime." - -"It's science fiction," Jackson said. - -"No, this is real." - -This statement was repeated several times, by several agents. -Jackson's ominously accurate game had passed from pure, -obscure, small-scale fantasy into the impure, highly publicized, -large-scale fantasy of the Hacker Crackdown. - -No mention was made of the real reason for the search. -According to their search warrant, the raiders had expected -to find the E911 Document stored on Jackson's bulletin board system. -But that warrant was sealed; a procedure that most law enforcement agencies -will use only when lives are demonstrably in danger. The raiders' -true motives were not discovered until the Jackson search-warrant -was unsealed by his lawyers, many months later. The Secret Service, -and the Chicago Computer Fraud and Abuse Task Force, -said absolutely nothing to Steve Jackson about any threat -to the police 911 System. They said nothing about the Atlanta Three, -nothing about Phrack or Knight Lightning, nothing about Terminus. - -Jackson was left to believe that his computers had been seized because -he intended to publish a science fiction book that law enforcement -considered too dangerous to see print. - -This misconception was repeated again and again, for months, -to an ever-widening public audience. It was not the truth of the case; -but as months passed, and this misconception was publicly printed again -and again, it became one of the few publicly known "facts" about -the mysterious Hacker Crackdown. The Secret Service had seized a computer -to stop the publication of a cyberpunk science fiction book. - -The second section of this book, "The Digital Underground," -is almost finished now. We have become acquainted with all -the major figures of this case who actually belong to the -underground milieu of computer intrusion. We have some idea -of their history, their motives, their general modus operandi. -We now know, I hope, who they are, where they came from, -and more or less what they want. In the next section of this book, -"Law and Order," we will leave this milieu and directly enter the -world of America's computer-crime police. - -At this point, however, I have another figure to introduce: myself. - -My name is Bruce Sterling. I live in Austin, Texas, where I am -a science fiction writer by trade: specifically, a CYBERPUNK -science fiction writer. - -Like my "cyberpunk" colleagues in the U.S. and Canada, -I've never been entirely happy with this literary label-- -especially after it became a synonym for computer criminal. -But I did once edit a book of stories by my colleagues, -called Mirrorshades: the Cyberpunk Anthology, and I've -long been a writer of literary-critical cyberpunk manifestos. -I am not a "hacker" of any description, though I do have readers -in the digital underground. - -When the Steve Jackson Games seizure occurred, I naturally took -an intense interest. If "cyberpunk" books were being banned -by federal police in my own home town, I reasonably wondered -whether I myself might be next. Would my computer be seized -by the Secret Service? At the time, I was in possession -of an aging Apple IIe without so much as a hard disk. -If I were to be raided as an author of computer-crime manuals, -the loss of my feeble word-processor would likely provoke more -snickers than sympathy. - -I'd known Steve Jackson for many years. We knew -one another as colleagues, for we frequented -the same local science-fiction conventions. -I'd played Jackson games, and recognized his cleverness; -but he certainly had never struck me as a potential mastermind -of computer crime. - -I also knew a little about computer bulletin-board systems. -In the mid-1980s I had taken an active role in an Austin board -called "SMOF-BBS," one of the first boards dedicated to science fiction. -I had a modem, and on occasion I'd logged on to Illuminati, -which always looked entertainly wacky, but certainly harmless enough. - -At the time of the Jackson seizure, I had no experience -whatsoever with underground boards. But I knew that no one -on Illuminati talked about breaking into systems illegally, -or about robbing phone companies. Illuminati didn't even -offer pirated computer games. Steve Jackson, like many creative artists, -was markedly touchy about theft of intellectual property. - -It seemed to me that Jackson was either seriously suspected -of some crime--in which case, he would be charged soon, -and would have his day in court--or else he was innocent, -in which case the Secret Service would quickly return his equipment, -and everyone would have a good laugh. I rather expected the good laugh. -The situation was not without its comic side. The raid, known -as the "Cyberpunk Bust" in the science fiction community, -was winning a great deal of free national publicity both -for Jackson himself and the "cyberpunk" science fiction -writers generally. - -Besides, science fiction people are used to being misinterpreted. -Science fiction is a colorful, disreputable, slipshod occupation, -full of unlikely oddballs, which, of course, is why we like it. -Weirdness can be an occupational hazard in our field. People who -wear Halloween costumes are sometimes mistaken for monsters. - -Once upon a time--back in 1939, in New York City-- -science fiction and the U.S. Secret Service collided in -a comic case of mistaken identity. This weird incident -involved a literary group quite famous in science fiction, -known as "the Futurians," whose membership included -such future genre greats as Isaac Asimov, Frederik Pohl, -and Damon Knight. The Futurians were every bit as -offbeat and wacky as any of their spiritual descendants, -including the cyberpunks, and were given to communal living, -spontaneous group renditions of light opera, and midnight fencing -exhibitions on the lawn. The Futurians didn't have bulletin -board systems, but they did have the technological equivalent -in 1939--mimeographs and a private printing press. These were -in steady use, producing a stream of science-fiction fan magazines, -literary manifestos, and weird articles, which were picked up -in ink-sticky bundles by a succession of strange, gangly, -spotty young men in fedoras and overcoats. - -The neighbors grew alarmed at the antics of the Futurians -and reported them to the Secret Service as suspected counterfeiters. -In the winter of 1939, a squad of USSS agents with drawn guns burst into -"Futurian House," prepared to confiscate the forged currency and illicit -printing presses. There they discovered a slumbering science fiction fan -named George Hahn, a guest of the Futurian commune who had just arrived -in New York. George Hahn managed to explain himself and his group, -and the Secret Service agents left the Futurians in peace henceforth. -(Alas, Hahn died in 1991, just before I had discovered this astonishing -historical parallel, and just before I could interview him for this book.) - -But the Jackson case did not come to a swift and comic end. -No quick answers came his way, or mine; no swift reassurances -that all was right in the digital world, that matters were well -in hand after all. Quite the opposite. In my alternate role -as a sometime pop-science journalist, I interviewed Jackson -and his staff for an article in a British magazine. -The strange details of the raid left me more concerned than ever. -Without its computers, the company had been financially -and operationally crippled. Half the SJG workforce, -a group of entirely innocent people, had been sorrowfully fired, -deprived of their livelihoods by the seizure. It began to dawn on me -that authors--American writers--might well have their computers seized, -under sealed warrants, without any criminal charge; and that, -as Steve Jackson had discovered, there was no immediate recourse for this. -This was no joke; this wasn't science fiction; this was real. - -I determined to put science fiction aside until I had discovered -what had happened and where this trouble had come from. -It was time to enter the purportedly real world of electronic -free expression and computer crime. Hence, this book. -Hence, the world of the telcos; and the world of the digital underground; -and next, the world of the police. - - - -PART THREE: LAW AND ORDER - - -Of the various anti-hacker activities of 1990, "Operation Sundevil" -had by far the highest public profile. The sweeping, nationwide -computer seizures of May 8, 1990 were unprecedented in scope and highly, -if rather selectively, publicized. - -Unlike the efforts of the Chicago Computer Fraud and Abuse Task Force, -"Operation Sundevil" was not intended to combat "hacking" in the sense -of computer intrusion or sophisticated raids on telco switching stations. -Nor did it have anything to do with hacker misdeeds with AT&T's software, -or with Southern Bell's proprietary documents. - -Instead, "Operation Sundevil" was a crackdown on those traditional scourges -of the digital underground: credit-card theft and telephone code abuse. -The ambitious activities out of Chicago, and the somewhat lesser-known -but vigorous anti-hacker actions of the New York State Police in 1990, -were never a part of "Operation Sundevil" per se, which was based in Arizona. - -Nevertheless, after the spectacular May 8 raids, the public, misled by -police secrecy, hacker panic, and a puzzled national press-corps, -conflated all aspects of the nationwide crackdown in 1990 under -the blanket term "Operation Sundevil." "Sundevil" is still the best-known -synonym for the crackdown of 1990. But the Arizona organizers of "Sundevil" -did not really deserve this reputation--any more, for instance, than all -hackers deserve a reputation as "hackers." - -There was some justice in this confused perception, though. -For one thing, the confusion was abetted by the Washington office -of the Secret Service, who responded to Freedom of Information Act -requests on "Operation Sundevil" by referring investigators -to the publicly known cases of Knight Lightning and the Atlanta Three. -And "Sundevil" was certainly the largest aspect of the Crackdown, -the most deliberate and the best-organized. As a crackdown on electronic -fraud, "Sundevil" lacked the frantic pace of the war on the Legion of Doom; -on the contrary, Sundevil's targets were picked out with cool deliberation -over an elaborate investigation lasting two full years. - -And once again the targets were bulletin board systems. - -Boards can be powerful aids to organized fraud. Underground boards carry -lively, extensive, detailed, and often quite flagrant "discussions" of -lawbreaking techniques and lawbreaking activities. "Discussing" crime -in the abstract, or "discussing" the particulars of criminal cases, -is not illegal--but there are stern state and federal laws against -coldbloodedly conspiring in groups in order to commit crimes. - -In the eyes of police, people who actively conspire to break the law -are not regarded as "clubs," "debating salons," "users' groups," or -"free speech advocates." Rather, such people tend to find themselves -formally indicted by prosecutors as "gangs," "racketeers," "corrupt -organizations" and "organized crime figures." - -What's more, the illicit data contained on outlaw boards goes well beyond -mere acts of speech and/or possible criminal conspiracy. As we have seen, -it was common practice in the digital underground to post purloined telephone -codes on boards, for any phreak or hacker who cared to abuse them. Is posting -digital booty of this sort supposed to be protected by the First Amendment? -Hardly--though the issue, like most issues in cyberspace, is not entirely -resolved. Some theorists argue that to merely RECITE a number publicly -is not illegal--only its USE is illegal. But anti-hacker police point out -that magazines and newspapers (more traditional forms of free expression) -never publish stolen telephone codes (even though this might well -raise their circulation). - -Stolen credit card numbers, being riskier and more valuable, -were less often publicly posted on boards--but there is no question -that some underground boards carried "carding" traffic, -generally exchanged through private mail. - -Underground boards also carried handy programs for "scanning" telephone -codes and raiding credit card companies, as well as the usual obnoxious -galaxy of pirated software, cracked passwords, blue-box schematics, -intrusion manuals, anarchy files, porn files, and so forth. - -But besides their nuisance potential for the spread of illicit knowledge, -bulletin boards have another vitally interesting aspect for the -professional investigator. Bulletin boards are cram-full of EVIDENCE. -All that busy trading of electronic mail, all those hacker boasts, -brags and struts, even the stolen codes and cards, can be neat, -electronic, real-time recordings of criminal activity. -As an investigator, when you seize a pirate board, you have -scored a coup as effective as tapping phones or intercepting mail. -However, you have not actually tapped a phone or intercepted a letter. -The rules of evidence regarding phone-taps and mail interceptions are old, -stern and well-understood by police, prosecutors and defense attorneys alike. -The rules of evidence regarding boards are new, waffling, and understood -by nobody at all. - -Sundevil was the largest crackdown on boards in world history. -On May 7, 8, and 9, 1990, about forty-two computer systems were seized. -Of those forty-two computers, about twenty-five actually were running boards. -(The vagueness of this estimate is attributable to the vagueness of -(a) what a "computer system" is, and (b) what it actually means to -"run a board" with one--or with two computers, or with three.) - -About twenty-five boards vanished into police custody in May 1990. -As we have seen, there are an estimated 30,000 boards in America today. -If we assume that one board in a hundred is up to no good with codes -and cards (which rather flatters the honesty of the board-using community), -then that would leave 2,975 outlaw boards untouched by Sundevil. -Sundevil seized about one tenth of one percent of all computer -bulletin boards in America. Seen objectively, this is something less -than a comprehensive assault. In 1990, Sundevil's organizers-- -the team at the Phoenix Secret Service office, and the Arizona -Attorney General's office-- had a list of at least THREE HUNDRED -boards that they considered fully deserving of search and seizure warrants. -The twenty-five boards actually seized were merely among the most obvious -and egregious of this much larger list of candidates. All these boards -had been examined beforehand--either by informants, who had passed printouts -to the Secret Service, or by Secret Service agents themselves, who not only -come equipped with modems but know how to use them. - -There were a number of motives for Sundevil. First, it offered -a chance to get ahead of the curve on wire-fraud crimes. -Tracking back credit-card ripoffs to their perpetrators -can be appallingly difficult. If these miscreants -have any kind of electronic sophistication, they can snarl -their tracks through the phone network into a mind-boggling, -untraceable mess, while still managing to "reach out and rob someone." -Boards, however, full of brags and boasts, codes and cards, -offer evidence in the handy congealed form. - -Seizures themselves--the mere physical removal of machines-- -tends to take the pressure off. During Sundevil, a large number -of code kids, warez d00dz, and credit card thieves would be deprived -of those boards--their means of community and conspiracy--in one swift blow. -As for the sysops themselves (commonly among the boldest offenders) -they would be directly stripped of their computer equipment, -and rendered digitally mute and blind. - -And this aspect of Sundevil was carried out with great success. -Sundevil seems to have been a complete tactical surprise-- -unlike the fragmentary and continuing seizures of the war on the -Legion of Doom, Sundevil was precisely timed and utterly overwhelming. -At least forty "computers" were seized during May 7, 8 and 9, 1990, -in Cincinnati, Detroit, Los Angeles, Miami, Newark, Phoenix, Tucson, -Richmond, San Diego, San Jose, Pittsburgh and San Francisco. -Some cities saw multiple raids, such as the five separate raids -in the New York City environs. Plano, Texas (essentially a suburb of -the Dallas/Fort Worth metroplex, and a hub of the telecommunications industry) -saw four computer seizures. Chicago, ever in the forefront, saw its own -local Sundevil raid, briskly carried out by Secret Service agents -Timothy Foley and Barbara Golden. - -Many of these raids occurred, not in the cities proper, -but in associated white-middle class suburbs--places like -Mount Lebanon, Pennsylvania and Clark Lake, Michigan. -There were a few raids on offices; most took place in people's homes, -the classic hacker basements and bedrooms. - -The Sundevil raids were searches and seizures, not a group of mass arrests. -There were only four arrests during Sundevil. "Tony the Trashman," -a longtime teenage bete noire of the Arizona Racketeering unit, -was arrested in Tucson on May 9. "Dr. Ripco," sysop of an outlaw board -with the misfortune to exist in Chicago itself, was also arrested-- -on illegal weapons charges. Local units also arrested a 19-year-old -female phone phreak named "Electra" in Pennsylvania, and a male juvenile -in California. Federal agents however were not seeking arrests, but computers. - -Hackers are generally not indicted (if at all) until the evidence -in their seized computers is evaluated--a process that can take weeks, -months--even years. When hackers are arrested on the spot, it's generally -an arrest for other reasons. Drugs and/or illegal weapons show up in a good -third of anti-hacker computer seizures (though not during Sundevil). - -That scofflaw teenage hackers (or their parents) should have marijuana -in their homes is probably not a shocking revelation, but the surprisingly -common presence of illegal firearms in hacker dens is a bit disquieting. -A Personal Computer can be a great equalizer for the techno-cowboy-- -much like that more traditional American "Great Equalizer," -the Personal Sixgun. Maybe it's not all that surprising -that some guy obsessed with power through illicit technology -would also have a few illicit high-velocity-impact devices around. -An element of the digital underground particularly dotes on those -"anarchy philes," and this element tends to shade into the crackpot milieu -of survivalists, gun-nuts, anarcho-leftists and the ultra-libertarian -right-wing. - -This is not to say that hacker raids to date have uncovered any -major crack-dens or illegal arsenals; but Secret Service agents -do not regard "hackers" as "just kids." They regard hackers as -unpredictable people, bright and slippery. It doesn't help matters -that the hacker himself has been "hiding behind his keyboard" -all this time. Commonly, police have no idea what he looks like. -This makes him an unknown quantity, someone best treated with -proper caution. - -To date, no hacker has come out shooting, though they do sometimes brag on -boards that they will do just that. Threats of this sort are taken seriously. -Secret Service hacker raids tend to be swift, comprehensive, well-manned -(even over-manned); and agents generally burst through every door -in the home at once, sometimes with drawn guns. Any potential resistance -is swiftly quelled. Hacker raids are usually raids on people's homes. -It can be a very dangerous business to raid an American home; -people can panic when strangers invade their sanctum. Statistically speaking, -the most dangerous thing a policeman can do is to enter someone's home. -(The second most dangerous thing is to stop a car in traffic.) -People have guns in their homes. More cops are hurt in homes -than are ever hurt in biker bars or massage parlors. - -But in any case, no one was hurt during Sundevil, -or indeed during any part of the Hacker Crackdown. - -Nor were there any allegations of any physical mistreatment of a suspect. -Guns were pointed, interrogations were sharp and prolonged; but no one -in 1990 claimed any act of brutality by any crackdown raider. - -In addition to the forty or so computers, Sundevil reaped floppy disks -in particularly great abundance--an estimated 23,000 of them, which -naturally included every manner of illegitimate data: pirated games, -stolen codes, hot credit card numbers, the complete text and software -of entire pirate bulletin-boards. These floppy disks, which remain -in police custody today, offer a gigantic, almost embarrassingly -rich source of possible criminal indictments. These 23,000 floppy disks -also include a thus-far unknown quantity of legitimate computer games, -legitimate software, purportedly "private" mail from boards, -business records, and personal correspondence of all kinds. - -Standard computer-crime search warrants lay great emphasis on seizing -written documents as well as computers--specifically including photocopies, -computer printouts, telephone bills, address books, logs, notes, -memoranda and correspondence. In practice, this has meant that diaries, -gaming magazines, software documentation, nonfiction books on hacking -and computer security, sometimes even science fiction novels, have all -vanished out the door in police custody. A wide variety of electronic items -have been known to vanish as well, including telephones, televisions, answering -machines, Sony Walkmans, desktop printers, compact disks, and audiotapes. - -No fewer than 150 members of the Secret Service were sent into -the field during Sundevil. They were commonly accompanied by -squads of local and/or state police. Most of these officers-- -especially the locals--had never been on an anti-hacker raid before. -(This was one good reason, in fact, why so many of them were invited along -in the first place.) Also, the presence of a uniformed police officer -assures the raidees that the people entering their homes are, in fact, police. -Secret Service agents wear plain clothes. So do the telco security experts -who commonly accompany the Secret Service on raids (and who make no particular -effort to identify themselves as mere employees of telephone companies). - -A typical hacker raid goes something like this. First, police storm in -rapidly, through every entrance, with overwhelming force, -in the assumption that this tactic will keep casualties to a minimum. -Second, possible suspects are immediately removed from the vicinity -of any and all computer systems, so that they will have no chance -to purge or destroy computer evidence. Suspects are herded into a room -without computers, commonly the living room, and kept under guard-- -not ARMED guard, for the guns are swiftly holstered, but under guard -nevertheless. They are presented with the search warrant and warned -that anything they say may be held against them. Commonly they have -a great deal to say, especially if they are unsuspecting parents. - -Somewhere in the house is the "hot spot"--a computer tied to a phone -line (possibly several computers and several phones). Commonly it's -a teenager's bedroom, but it can be anywhere in the house; -there may be several such rooms. This "hot spot" is put in charge -of a two-agent team, the "finder" and the "recorder." The "finder" -is computer-trained, commonly the case agent who has actually obtained -the search warrant from a judge. He or she understands what is being sought, -and actually carries out the seizures: unplugs machines, opens drawers, -desks, files, floppy-disk containers, etc. The "recorder" photographs -all the equipment, just as it stands--especially the tangle of -wired connections in the back, which can otherwise be a real nightmare -to restore. The recorder will also commonly photograph every room -in the house, lest some wily criminal claim that the police had robbed him -during the search. Some recorders carry videocams or tape recorders; -however, it's more common for the recorder to simply take written notes. -Objects are described and numbered as the finder seizes them, generally -on standard preprinted police inventory forms. - -Even Secret Service agents were not, and are not, expert computer users. -They have not made, and do not make, judgements on the fly about potential -threats posed by various forms of equipment. They may exercise discretion; -they may leave Dad his computer, for instance, but they don't HAVE to. -Standard computer-crime search warrants, which date back to the early 80s, -use a sweeping language that targets computers, most anything attached -to a computer, most anything used to operate a computer--most anything -that remotely resembles a computer--plus most any and all written documents -surrounding it. Computer-crime investigators have strongly urged agents -to seize the works. - -In this sense, Operation Sundevil appears to have been a complete success. -Boards went down all over America, and were shipped en masse to the computer -investigation lab of the Secret Service, in Washington DC, along with the -23,000 floppy disks and unknown quantities of printed material. - -But the seizure of twenty-five boards, and the multi-megabyte mountains -of possibly useful evidence contained in these boards (and in their owners' -other computers, also out the door), were far from the only motives for -Operation Sundevil. An unprecedented action of great ambition and size, -Sundevil's motives can only be described as political. It was a -public-relations effort, meant to pass certain messages, meant to make -certain situations clear: both in the mind of the general public, -and in the minds of various constituencies of the electronic community. - - First --and this motivation was vital--a "message" would be sent from -law enforcement to the digital underground. This very message was recited -in so many words by Garry M. Jenkins, the Assistant Director of the -US Secret Service, at the Sundevil press conference in Phoenix on -May 9, 1990, immediately after the raids. In brief, hackers were -mistaken in their foolish belief that they could hide behind the -"relative anonymity of their computer terminals." On the contrary, -they should fully understand that state and federal cops were -actively patrolling the beat in cyberspace--that they were -on the watch everywhere, even in those sleazy and secretive -dens of cybernetic vice, the underground boards. - -This is not an unusual message for police to publicly convey to crooks. -The message is a standard message; only the context is new. - -In this respect, the Sundevil raids were the digital equivalent -of the standard vice-squad crackdown on massage parlors, porno bookstores, -head-shops, or floating crap-games. There may be few or no arrests in a raid -of this sort; no convictions, no trials, no interrogations. In cases of this -sort, police may well walk out the door with many pounds of sleazy magazines, -X-rated videotapes, sex toys, gambling equipment, baggies of marijuana. . . . - -Of course, if something truly horrendous is discovered by the raiders, -there will be arrests and prosecutions. Far more likely, however, -there will simply be a brief but sharp disruption of the closed -and secretive world of the nogoodniks. There will be "street hassle." -"Heat." "Deterrence." And, of course, the immediate loss of the seized goods. -It is very unlikely that any of this seized material will ever be returned. -Whether charged or not, whether convicted or not, the perpetrators will -almost surely lack the nerve ever to ask for this stuff to be given back. - -Arrests and trials--putting people in jail--may involve all kinds of -formal legalities; but dealing with the justice system is far from the only -task of police. Police do not simply arrest people. They don't simply -put people in jail. That is not how the police perceive their jobs. -Police "protect and serve." Police "keep the peace," they "keep public order." -Like other forms of public relations, keeping public order is not an -exact science. Keeping public order is something of an art-form. - -If a group of tough-looking teenage hoodlums was loitering on a street-corner, -no one would be surprised to see a street-cop arrive and sternly order -them to "break it up." On the contrary, the surprise would come if one -of these ne'er-do-wells stepped briskly into a phone-booth, -called a civil rights lawyer, and instituted a civil suit -in defense of his Constitutional rights of free speech -and free assembly. But something much along this line -was one of the many anomolous outcomes of the Hacker Crackdown. - -Sundevil also carried useful "messages" for other constituents of -the electronic community. These messages may not have been read -aloud from the Phoenix podium in front of the press corps, -but there was little mistaking their meaning. There was a message -of reassurance for the primary victims of coding and carding: -the telcos, and the credit companies. Sundevil was greeted with joy -by the security officers of the electronic business community. -After years of high-tech harassment and spiralling revenue losses, -their complaints of rampant outlawry were being taken seriously by -law enforcement. No more head-scratching or dismissive shrugs; -no more feeble excuses about "lack of computer-trained officers" or -the low priority of "victimless" white-collar telecommunication crimes. - -Computer-crime experts have long believed that computer-related offenses -are drastically under-reported. They regard this as a major open scandal -of their field. Some victims are reluctant to come forth, because they -believe that police and prosecutors are not computer-literate, -and can and will do nothing. Others are embarrassed by -their vulnerabilities, and will take strong measures -to avoid any publicity; this is especially true of banks, -who fear a loss of investor confidence should an embezzlement-case -or wire-fraud surface. And some victims are so helplessly confused -by their own high technology that they never even realize that -a crime has occurred--even when they have been fleeced to the bone. - -The results of this situation can be dire. -Criminals escape apprehension and punishment. -The computer-crime units that do exist, can't get work. -The true scope of computer-crime: its size, its real nature, -the scope of its threats, and the legal remedies for it-- -all remain obscured. - -Another problem is very little publicized, but it is a cause -of genuine concern. Where there is persistent crime, -but no effective police protection, then vigilantism can result. -Telcos, banks, credit companies, the major corporations who -maintain extensive computer networks vulnerable to hacking ---these organizations are powerful, wealthy, and -politically influential. They are disinclined to be -pushed around by crooks (or by most anyone else, -for that matter). They often maintain well-organized -private security forces, commonly run by -experienced veterans of military and police units, -who have left public service for the greener pastures -of the private sector. For police, the corporate -security manager can be a powerful ally; but if this -gentleman finds no allies in the police, and the -pressure is on from his board-of-directors, -he may quietly take certain matters into his own hands. - -Nor is there any lack of disposable hired-help in the -corporate security business. Private security agencies-- -the `security business' generally--grew explosively in the 1980s. -Today there are spooky gumshoed armies of "security consultants," -"rent-a- cops," "private eyes," "outside experts"--every manner -of shady operator who retails in "results" and discretion. -Or course, many of these gentlemen and ladies may be paragons -of professional and moral rectitude. But as anyone -who has read a hard-boiled detective novel knows, -police tend to be less than fond of this sort -of private-sector competition. - -Companies in search of computer-security have even been -known to hire hackers. Police shudder at this prospect. - -Police treasure good relations with the business community. -Rarely will you see a policeman so indiscreet as to allege -publicly that some major employer in his state or city has succumbed -to paranoia and gone off the rails. Nevertheless, -police --and computer police in particular--are aware -of this possibility. Computer-crime police can and do -spend up to half of their business hours just doing -public relations: seminars, "dog and pony shows," -sometimes with parents' groups or computer users, -but generally with their core audience: the likely -victims of hacking crimes. These, of course, are telcos, -credit card companies and large computer-equipped corporations. -The police strongly urge these people, as good citizens, -to report offenses and press criminal charges; -they pass the message that there is someone in authority who cares, -understands, and, best of all, will take useful action -should a computer-crime occur. - -But reassuring talk is cheap. Sundevil offered action. - -The final message of Sundevil was intended for internal consumption -by law enforcement. Sundevil was offered as proof that the community -of American computer-crime police had come of age. Sundevil was -proof that enormous things like Sundevil itself could now be accomplished. -Sundevil was proof that the Secret Service and its local law-enforcement -allies could act like a well-oiled machine--(despite the hampering use -of those scrambled phones). It was also proof that the Arizona Organized -Crime and Racketeering Unit--the sparkplug of Sundevil--ranked with the best -in the world in ambition, organization, and sheer conceptual daring. - -And, as a final fillip, Sundevil was a message from the Secret Service -to their longtime rivals in the Federal Bureau of Investigation. -By Congressional fiat, both USSS and FBI formally share jurisdiction -over federal computer-crimebusting activities. Neither of these groups -has ever been remotely happy with this muddled situation. It seems to -suggest that Congress cannot make up its mind as to which of these groups -is better qualified. And there is scarcely a G-man or a Special Agent -anywhere without a very firm opinion on that topic. - -# - -For the neophyte, one of the most puzzling aspects of the crackdown -on hackers is why the United States Secret Service has anything at all -to do with this matter. - -The Secret Service is best known for its primary public role: -its agents protect the President of the United States. -They also guard the President's family, the Vice President and his family, -former Presidents, and Presidential candidates. They sometimes guard -foreign dignitaries who are visiting the United States, especially foreign -heads of state, and have been known to accompany American officials -on diplomatic missions overseas. - -Special Agents of the Secret Service don't wear uniforms, but the -Secret Service also has two uniformed police agencies. There's the -former White House Police (now known as the Secret Service Uniformed Division, -since they currently guard foreign embassies in Washington, as well as the -White House itself). And there's the uniformed Treasury Police Force. - -The Secret Service has been charged by Congress with a number -of little-known duties. They guard the precious metals in Treasury vaults. -They guard the most valuable historical documents of the United States: -originals of the Constitution, the Declaration of Independence, -Lincoln's Second Inaugural Address, an American-owned copy of -the Magna Carta, and so forth. Once they were assigned to guard -the Mona Lisa, on her American tour in the 1960s. - -The entire Secret Service is a division of the Treasury Department. -Secret Service Special Agents (there are about 1,900 of them) -are bodyguards for the President et al, but they all work for the Treasury. -And the Treasury (through its divisions of the U.S. Mint and the -Bureau of Engraving and Printing) prints the nation's money. - -As Treasury police, the Secret Service guards the nation's currency; -it is the only federal law enforcement agency with direct jurisdiction -over counterfeiting and forgery. It analyzes documents for authenticity, -and its fight against fake cash is still quite lively (especially since -the skilled counterfeiters of Medellin, Columbia have gotten into the act). -Government checks, bonds, and other obligations, which exist in untold -millions and are worth untold billions, are common targets for forgery, -which the Secret Service also battles. It even handles forgery -of postage stamps. - -But cash is fading in importance today as money has become electronic. -As necessity beckoned, the Secret Service moved from fighting the -counterfeiting of paper currency and the forging of checks, -to the protection of funds transferred by wire. - -From wire-fraud, it was a simple skip-and-jump to what is formally -known as "access device fraud." Congress granted the Secret Service -the authority to investigate "access device fraud" under Title 18 -of the United States Code (U.S.C. Section 1029). - -The term "access device" seems intuitively simple. It's some kind -of high-tech gizmo you use to get money with. It makes good sense -to put this sort of thing in the charge of counterfeiting and -wire-fraud experts. - -However, in Section 1029, the term "access device" is very -generously defined. An access device is: "any card, plate, -code, account number, or other means of account access -that can be used, alone or in conjunction with another access device, -to obtain money, goods, services, or any other thing of value, -or that can be used to initiate a transfer of funds." - -"Access device" can therefore be construed to include credit cards -themselves (a popular forgery item nowadays). It also includes credit card -account NUMBERS, those standards of the digital underground. The same goes -for telephone charge cards (an increasingly popular item with telcos, -who are tired of being robbed of pocket change by phone-booth thieves). -And also telephone access CODES, those OTHER standards of the digital -underground. (Stolen telephone codes may not "obtain money," but they -certainly do obtain valuable "services," which is specifically forbidden -by Section 1029.) - -We can now see that Section 1029 already pits the United States Secret Service -directly against the digital underground, without any mention at all of -the word "computer." - -Standard phreaking devices, like "blue boxes," used to steal phone service -from old-fashioned mechanical switches, are unquestionably "counterfeit -access devices." Thanks to Sec.1029, it is not only illegal to USE -counterfeit access devices, but it is even illegal to BUILD them. -"Producing," "designing" "duplicating" or "assembling" blue boxes -are all federal crimes today, and if you do this, the Secret Service -has been charged by Congress to come after you. - -Automatic Teller Machines, which replicated all over America during the 1980s, -are definitely "access devices," too, and an attempt to tamper with their -punch-in codes and plastic bank cards falls directly under Sec. 1029. - -Section 1029 is remarkably elastic. Suppose you find a computer password -in somebody's trash. That password might be a "code"--it's certainly a -"means of account access." Now suppose you log on to a computer -and copy some software for yourself. You've certainly obtained -"service" (computer service) and a "thing of value" (the software). -Suppose you tell a dozen friends about your swiped password, -and let them use it, too. Now you're "trafficking in unauthorized -access devices." And when the Prophet, a member of the Legion of Doom, -passed a stolen telephone company document to Knight Lightning -at Phrack magazine, they were both charged under Sec. 1029! - -There are two limitations on Section 1029. First, the offense must -"affect interstate or foreign commerce" in order to become a matter -of federal jurisdiction. The term "affecting commerce" is not well defined; -but you may take it as a given that the Secret Service can take an interest -if you've done most anything that happens to cross a state line. -State and local police can be touchy about their jurisdictions, -and can sometimes be mulish when the feds show up. But when it comes -to computer-crime, the local police are pathetically grateful -for federal help--in fact they complain that they can't get enough of it. -If you're stealing long-distance service, you're almost certainly crossing -state lines, and you're definitely "affecting the interstate commerce" -of the telcos. And if you're abusing credit cards by ordering stuff -out of glossy catalogs from, say, Vermont, you're in for it. - -The second limitation is money. As a rule, the feds don't pursue -penny-ante offenders. Federal judges will dismiss cases that appear -to waste their time. Federal crimes must be serious; Section 1029 -specifies a minimum loss of a thousand dollars. - -We now come to the very next section of Title 18, which is Section 1030, -"Fraud and related activity in connection with computers." This statute -gives the Secret Service direct jurisdiction over acts of computer intrusion. -On the face of it, the Secret Service would now seem to command the field. -Section 1030, however, is nowhere near so ductile as Section 1029. - -The first annoyance is Section 1030(d), which reads: - -"(d) The United States Secret Service shall, -IN ADDITION TO ANY OTHER AGENCY HAVING SUCH AUTHORITY, -have the authority to investigate offenses under this section. -Such authority of the United States Secret Service shall be -exercised in accordance with an agreement which shall be entered -into by the Secretary of the Treasury AND THE ATTORNEY GENERAL." -(Author's italics.) [Represented by capitals.] - -The Secretary of the Treasury is the titular head of the Secret Service, -while the Attorney General is in charge of the FBI. In Section (d), -Congress shrugged off responsibility for the computer-crime turf-battle -between the Service and the Bureau, and made them fight it out all -by themselves. The result was a rather dire one for the Secret Service, -for the FBI ended up with exclusive jurisdiction over computer break-ins -having to do with national security, foreign espionage, federally insured -banks, and U.S. military bases, while retaining joint jurisdiction over -all the other computer intrusions. Essentially, when it comes to Section 1030, -the FBI not only gets the real glamor stuff for itself, but can peer over the -shoulder of the Secret Service and barge in to meddle whenever it suits them. - -The second problem has to do with the dicey term -"Federal interest computer." Section 1030(a)(2) -makes it illegal to "access a computer without authorization" -if that computer belongs to a financial institution or an issuer -of credit cards (fraud cases, in other words). Congress was quite -willing to give the Secret Service jurisdiction over -money-transferring computers, but Congress balked at -letting them investigate any and all computer intrusions. -Instead, the USSS had to settle for the money machines -and the "Federal interest computers." A "Federal interest computer" -is a computer which the government itself owns, or is using. -Large networks of interstate computers, linked over state lines, -are also considered to be of "Federal interest." (This notion of -"Federal interest" is legally rather foggy and has never been -clearly defined in the courts. The Secret Service has never yet -had its hand slapped for investigating computer break-ins that were NOT -of "Federal interest," but conceivably someday this might happen.) - -So the Secret Service's authority over "unauthorized access" -to computers covers a lot of territory, but by no means the -whole ball of cyberspatial wax. If you are, for instance, -a LOCAL computer retailer, or the owner of a LOCAL bulletin -board system, then a malicious LOCAL intruder can break in, -crash your system, trash your files and scatter viruses, -and the U.S. Secret Service cannot do a single thing about it. - -At least, it can't do anything DIRECTLY. But the Secret Service -will do plenty to help the local people who can. - -The FBI may have dealt itself an ace off the bottom of the deck -when it comes to Section 1030; but that's not the whole story; -that's not the street. What's Congress thinks is one thing, -and Congress has been known to change its mind. The REAL -turf-struggle is out there in the streets where it's happening. -If you're a local street-cop with a computer problem, -the Secret Service wants you to know where you can find -the real expertise. While the Bureau crowd are off having -their favorite shoes polished--(wing-tips)--and making derisive -fun of the Service's favorite shoes--("pansy-ass tassels")-- -the tassel-toting Secret Service has a crew of ready-and-able -hacker-trackers installed in the capital of every state in the Union. -Need advice? They'll give you advice, or at least point you in -the right direction. Need training? They can see to that, too. - -If you're a local cop and you call in the FBI, the FBI -(as is widely and slanderously rumored) will order you around -like a coolie, take all the credit for your busts, -and mop up every possible scrap of reflected glory. -The Secret Service, on the other hand, doesn't brag a lot. -They're the quiet types. VERY quiet. Very cool. Efficient. -High-tech. Mirrorshades, icy stares, radio ear-plugs, -an Uzi machine-pistol tucked somewhere in that well-cut jacket. -American samurai, sworn to give their lives to protect our President. -"The granite agents." Trained in martial arts, absolutely fearless. -Every single one of 'em has a top-secret security clearance. -Something goes a little wrong, you're not gonna hear any whining -and moaning and political buck-passing out of these guys. - -The facade of the granite agent is not, of course, the reality. -Secret Service agents are human beings. And the real glory -in Service work is not in battling computer crime--not yet, -anyway--but in protecting the President. The real glamour -of Secret Service work is in the White House Detail. -If you're at the President's side, then the kids and the wife -see you on television; you rub shoulders with the most powerful -people in the world. That's the real heart of Service work, -the number one priority. More than one computer investigation -has stopped dead in the water when Service agents vanished at -the President's need. - -There's romance in the work of the Service. The intimate access -to circles of great power; the esprit-de-corps of a highly trained -and disciplined elite; the high responsibility of defending the -Chief Executive; the fulfillment of a patriotic duty. And as police -work goes, the pay's not bad. But there's squalor in Service work, too. -You may get spat upon by protesters howling abuse--and if they get violent, -if they get too close, sometimes you have to knock one of them down-- -discreetly. - -The real squalor in Service work is drudgery such as "the quarterlies," -traipsing out four times a year, year in, year out, to interview the various -pathetic wretches, many of them in prisons and asylums, who have seen fit -to threaten the President's life. And then there's the grinding stress -of searching all those faces in the endless bustling crowds, looking for -hatred, looking for psychosis, looking for the tight, nervous face -of an Arthur Bremer, a Squeaky Fromme, a Lee Harvey Oswald. -It's watching all those grasping, waving hands for sudden movements, -while your ears strain at your radio headphone for the long-rehearsed -cry of "Gun!" - -It's poring, in grinding detail, over the biographies of every rotten -loser who ever shot at a President. It's the unsung work of the -Protective Research Section, who study scrawled, anonymous death threats -with all the meticulous tools of anti-forgery techniques. - -And it's maintaining the hefty computerized files on anyone -who ever threatened the President's life. Civil libertarians -have become increasingly concerned at the Government's use -of computer files to track American citizens--but the -Secret Service file of potential Presidential assassins, -which has upward of twenty thousand names, rarely causes -a peep of protest. If you EVER state that you intend to -kill the President, the Secret Service will want to know -and record who you are, where you are, what you are, -and what you're up to. If you're a serious threat-- -if you're officially considered "of protective interest"-- -then the Secret Service may well keep tabs on you -for the rest of your natural life. - -Protecting the President has first call on all the Service's resources. -But there's a lot more to the Service's traditions and history than -standing guard outside the Oval Office. - -The Secret Service is the nation's oldest general federal -law-enforcement agency. Compared to the Secret Service, -the FBI are new-hires and the CIA are temps. The Secret Service -was founded 'way back in 1865, at the suggestion of Hugh McCulloch, -Abraham Lincoln's Secretary of the Treasury. McCulloch wanted -a specialized Treasury police to combat counterfeiting. -Abraham Lincoln agreed that this seemed a good idea, and, -with a terrible irony, Abraham Lincoln was shot that -very night by John Wilkes Booth. - -The Secret Service originally had nothing to do with protecting Presidents. -They didn't take this on as a regular assignment until after the Garfield -assassination in 1881. And they didn't get any Congressional money for it -until President McKinley was shot in 1901. The Service was originally -designed for one purpose: destroying counterfeiters. - -# - -There are interesting parallels between the Service's -nineteenth-century entry into counterfeiting, -and America's twentieth-century entry into computer-crime. - -In 1865, America's paper currency was a terrible muddle. -Security was drastically bad. Currency was printed on the spot -by local banks in literally hundreds of different designs. -No one really knew what the heck a dollar bill was supposed to look like. -Bogus bills passed easily. If some joker told you that a one-dollar bill -from the Railroad Bank of Lowell, Massachusetts had a woman leaning on -a shield, with a locomotive, a cornucopia, a compass, various agricultural -implements, a railroad bridge, and some factories, then you pretty much had -to take his word for it. (And in fact he was telling the truth!) - -SIXTEEN HUNDRED local American banks designed and printed their own -paper currency, and there were no general standards for security. -Like a badly guarded node in a computer network, badly designed bills -were easy to fake, and posed a security hazard for the entire monetary system. - -No one knew the exact extent of the threat to the currency. -There were panicked estimates that as much as a third of -the entire national currency was faked. Counterfeiters-- -known as "boodlers" in the underground slang of the time-- -were mostly technically skilled printers who had gone to the bad. -Many had once worked printing legitimate currency. -Boodlers operated in rings and gangs. Technical experts -engraved the bogus plates--commonly in basements in New York City. -Smooth confidence men passed large wads of high-quality, -high-denomination fakes, including the really sophisticated stuff-- -government bonds, stock certificates, and railway shares. -Cheaper, botched fakes were sold or sharewared to low-level -gangs of boodler wannabes. (The really cheesy lowlife boodlers -merely upgraded real bills by altering face values, -changing ones to fives, tens to hundreds, and so on.) - -The techniques of boodling were little-known and regarded -with a certain awe by the mid- nineteenth-century public. -The ability to manipulate the system for rip-off seemed -diabolically clever. As the skill and daring of the -boodlers increased, the situation became intolerable. -The federal government stepped in, and began offering -its own federal currency, which was printed in fancy green ink, -but only on the back--the original "greenbacks." And at first, -the improved security of the well-designed, well-printed -federal greenbacks seemed to solve the problem; but then -the counterfeiters caught on. Within a few years things were -worse than ever: a CENTRALIZED system where ALL security was bad! - -The local police were helpless. The Government tried offering -blood money to potential informants, but this met with little success. -Banks, plagued by boodling, gave up hope of police help and hired -private security men instead. Merchants and bankers queued up -by the thousands to buy privately-printed manuals on currency security, -slim little books like Laban Heath's INFALLIBLE GOVERNMENT -COUNTERFEIT DETECTOR. The back of the book offered Laban Heath's -patent microscope for five bucks. - -Then the Secret Service entered the picture. The first agents -were a rough and ready crew. Their chief was one William P. Wood, -a former guerilla in the Mexican War who'd won a reputation busting -contractor fraudsters for the War Department during the Civil War. -Wood, who was also Keeper of the Capital Prison, had a sideline -as a counterfeiting expert, bagging boodlers for the federal bounty money. - -Wood was named Chief of the new Secret Service in July 1865. -There were only ten Secret Service agents in all: Wood himself, -a handful who'd worked for him in the War Department, and a few -former private investigators--counterfeiting experts--whom Wood -had won over to public service. (The Secret Service of 1865 was -much the size of the Chicago Computer Fraud Task Force or the -Arizona Racketeering Unit of 1990.) These ten "Operatives" -had an additional twenty or so "Assistant Operatives" and "Informants." -Besides salary and per diem, each Secret Service employee received -a whopping twenty-five dollars for each boodler he captured. - -Wood himself publicly estimated that at least HALF of America's currency -was counterfeit, a perhaps pardonable perception. Within a year the -Secret Service had arrested over 200 counterfeiters. They busted about -two hundred boodlers a year for four years straight. - -Wood attributed his success to travelling fast and light, hitting the -bad-guys hard, and avoiding bureaucratic baggage. "Because my raids -were made without military escort and I did not ask the assistance -of state officers, I surprised the professional counterfeiter." - -Wood's social message to the once-impudent boodlers bore an eerie ring -of Sundevil: "It was also my purpose to convince such characters that -it would no longer be healthy for them to ply their vocation without -being handled roughly, a fact they soon discovered." - -William P. Wood, the Secret Service's guerilla pioneer, -did not end well. He succumbed to the lure of aiming for -the really big score. The notorious Brockway Gang of New York City, -headed by William E. Brockway, the "King of the Counterfeiters," -had forged a number of government bonds. They'd passed these -brilliant fakes on the prestigious Wall Street investment -firm of Jay Cooke and Company. The Cooke firm were frantic -and offered a huge reward for the forgers' plates. - -Laboring diligently, Wood confiscated the plates -(though not Mr. Brockway) and claimed the reward. -But the Cooke company treacherously reneged. -Wood got involved in a down-and-dirty lawsuit -with the Cooke capitalists. Wood's boss, -Secretary of the Treasury McCulloch, felt that -Wood's demands for money and glory were unseemly, -and even when the reward money finally came through, -McCulloch refused to pay Wood anything. -Wood found himself mired in a seemingly endless -round of federal suits and Congressional lobbying. - -Wood never got his money. And he lost his job to boot. -He resigned in 1869. - -Wood's agents suffered, too. On May 12, 1869, the second Chief -of the Secret Service took over, and almost immediately fired -most of Wood's pioneer Secret Service agents: Operatives, -Assistants and Informants alike. The practice of receiving $25 -per crook was abolished. And the Secret Service began the long, -uncertain process of thorough professionalization. - -Wood ended badly. He must have felt stabbed in the back. -In fact his entire organization was mangled. - -On the other hand, William P. Wood WAS the first head of the Secret Service. -William Wood was the pioneer. People still honor his name. Who remembers -the name of the SECOND head of the Secret Service? - -As for William Brockway (also known as "Colonel Spencer"), -he was finally arrested by the Secret Service in 1880. -He did five years in prison, got out, and was still boodling -at the age of seventy-four. - -# - -Anyone with an interest in Operation Sundevil-- -or in American computer-crime generally-- -could scarcely miss the presence of Gail Thackeray, -Assistant Attorney General of the State of Arizona. -Computer-crime training manuals often cited -Thackeray's group and her work; she was the -highest-ranking state official to specialize -in computer-related offenses. Her name had been -on the Sundevil press release (though modestly ranked -well after the local federal prosecuting attorney and -the head of the Phoenix Secret Service office). - -As public commentary, and controversy, began to mount -about the Hacker Crackdown, this Arizonan state official -began to take a higher and higher public profile. -Though uttering almost nothing specific about -the Sundevil operation itself, she coined some -of the most striking soundbites of the growing propaganda war: -"Agents are operating in good faith, and I don't think -you can say that for the hacker community," was one. -Another was the memorable "I am not a mad dog prosecutor" -(Houston Chronicle, Sept 2, 1990.) In the meantime, -the Secret Service maintained its usual extreme discretion; -the Chicago Unit, smarting from the backlash -of the Steve Jackson scandal, had gone completely to earth. - -As I collated my growing pile of newspaper clippings, -Gail Thackeray ranked as a comparative fount of public -knowledge on police operations. - -I decided that I had to get to know Gail Thackeray. -I wrote to her at the Arizona Attorney General's Office. -Not only did she kindly reply to me, but, to my astonishment, -she knew very well what "cyberpunk" science fiction was. - -Shortly after this, Gail Thackeray lost her job. -And I temporarily misplaced my own career as -a science-fiction writer, to become a full-time -computer-crime journalist. In early March, 1991, -I flew to Phoenix, Arizona, to interview Gail Thackeray -for my book on the hacker crackdown. - -# - -"Credit cards didn't used to cost anything to get," -says Gail Thackeray. "Now they cost forty bucks-- -and that's all just to cover the costs from RIP-OFF ARTISTS." - -Electronic nuisance criminals are parasites. -One by one they're not much harm, no big deal. -But they never come just one by one. They come in swarms, -heaps, legions, sometimes whole subcultures. And they bite. -Every time we buy a credit card today, we lose a little financial -vitality to a particular species of bloodsucker. - -What, in her expert opinion, are the worst forms of electronic crime, -I ask, consulting my notes. Is it--credit card fraud? Breaking into -ATM bank machines? Phone-phreaking? Computer intrusions? -Software viruses? Access-code theft? Records tampering? -Software piracy? Pornographic bulletin boards? -Satellite TV piracy? Theft of cable service? -It's a long list. By the time I reach the end -of it I feel rather depressed. - -"Oh no," says Gail Thackeray, leaning forward over the table, -her whole body gone stiff with energetic indignation, -"the biggest damage is telephone fraud. Fake sweepstakes, -fake charities. Boiler-room con operations. You could pay off -the national debt with what these guys steal. . . . -They target old people, they get hold of credit ratings -and demographics, they rip off the old and the weak." -The words come tumbling out of her. - -It's low-tech stuff, your everyday boiler-room fraud. -Grifters, conning people out of money over the phone, -have been around for decades. This is where the word "phony" came from! - -It's just that it's so much EASIER now, horribly facilitated by advances -in technology and the byzantine structure of the modern phone system. -The same professional fraudsters do it over and over, Thackeray tells me, -they hide behind dense onion-shells of fake companies. . . fake holding -corporations nine or ten layers deep, registered all over the map. -They get a phone installed under a false name in an empty safe-house. -And then they call-forward everything out of that phone to yet -another phone, a phone that may even be in another STATE. -And they don't even pay the charges on their phones; -after a month or so, they just split; set up somewhere else -in another Podunkville with the same seedy crew of veteran phone-crooks. -They buy or steal commercial credit card reports, slap them on the PC, -have a program pick out people over sixty-five who pay a lot to charities. -A whole subculture living off this, merciless folks on the con. - -"The `light-bulbs for the blind' people," Thackeray muses, -with a special loathing. "There's just no end to them." - -We're sitting in a downtown diner in Phoenix, Arizona. -It's a tough town, Phoenix. A state capital seeing some hard times. -Even to a Texan like myself, Arizona state politics seem rather baroque. -There was, and remains, endless trouble over the Martin Luther King holiday, -the sort of stiff-necked, foot-shooting incident for which Arizona politics -seem famous. There was Evan Mecham, the eccentric Republican millionaire -governor who was impeached, after reducing state government to a -ludicrous shambles. Then there was the national Keating scandal, -involving Arizona savings and loans, in which both of Arizona's -U.S. senators, DeConcini and McCain, played sadly prominent roles. - -And the very latest is the bizarre AzScam case, -in which state legislators were videotaped, -eagerly taking cash from an informant of the Phoenix city -police department, who was posing as a Vegas mobster. - -"Oh," says Thackeray cheerfully. "These people are amateurs here, -they thought they were finally getting to play with the big boys. -They don't have the least idea how to take a bribe! -It's not institutional corruption. It's not like back in Philly." - -Gail Thackeray was a former prosecutor in Philadelphia. -Now she's a former assistant attorney general of the State of Arizona. -Since moving to Arizona in 1986, she had worked under the aegis -of Steve Twist, her boss in the Attorney General's office. -Steve Twist wrote Arizona's pioneering computer crime laws -and naturally took an interest in seeing them enforced. -It was a snug niche, and Thackeray's Organized Crime and -Racketeering Unit won a national reputation for ambition -and technical knowledgeability. . . . Until the latest -election in Arizona. Thackeray's boss ran for the top -job, and lost. The victor, the new Attorney General, -apparently went to some pains to eliminate the bureaucratic -traces of his rival, including his pet group--Thackeray's group. -Twelve people got their walking papers. - -Now Thackeray's painstakingly assembled computer lab -sits gathering dust somewhere in the glass-and-concrete -Attorney General's HQ on 1275 Washington Street. -Her computer-crime books, her painstakingly garnered -back issues of phreak and hacker zines, all bought -at her own expense--are piled in boxes somewhere. -The State of Arizona is simply not particularly -interested in electronic racketeering at the moment. - -At the moment of our interview, Gail Thackeray, -officially unemployed, is working out of the county -sheriff's office, living on her savings, and prosecuting -several cases--working 60-hour weeks, just as always-- -for no pay at all. "I'm trying to train people," -she mutters. - -Half her life seems to be spent training people--merely pointing out, -to the naive and incredulous (such as myself) that this stuff -is ACTUALLY GOING ON OUT THERE. It's a small world, computer crime. -A young world. Gail Thackeray, a trim blonde Baby-Boomer who favors -Grand Canyon white-water rafting to kill some slow time, -is one of the world's most senior, most veteran "hacker-trackers." -Her mentor was Donn Parker, the California think-tank theorist -who got it all started `way back in the mid-70s, the "grandfather -of the field," "the great bald eagle of computer crime." - -And what she has learned, Gail Thackeray teaches. Endlessly. -Tirelessly. To anybody. To Secret Service agents and state police, -at the Glynco, Georgia federal training center. To local police, -on "roadshows" with her slide projector and notebook. -To corporate security personnel. To journalists. To parents. - -Even CROOKS look to Gail Thackeray for advice. -Phone-phreaks call her at the office. They know very -well who she is. They pump her for information -on what the cops are up to, how much they know. -Sometimes whole CROWDS of phone phreaks, -hanging out on illegal conference calls, will call Gail -Thackeray up. They taunt her. And, as always, -they boast. Phone-phreaks, real stone phone-phreaks, -simply CANNOT SHUT UP. They natter on for hours. - -Left to themselves, they mostly talk about the intricacies -of ripping-off phones; it's about as interesting as listening -to hot-rodders talk about suspension and distributor-caps. -They also gossip cruelly about each other. And when talking -to Gail Thackeray, they incriminate themselves. "I have tapes," -Thackeray says coolly. - -Phone phreaks just talk like crazy. "Dial-Tone" out in Alabama -has been known to spend half-an-hour simply reading stolen -phone-codes aloud into voice-mail answering machines. -Hundreds, thousands of numbers, recited in a monotone, -without a break--an eerie phenomenon. When arrested, -it's a rare phone phreak who doesn't inform at endless length -on everybody he knows. - -Hackers are no better. What other group of criminals, -she asks rhetorically, publishes newsletters and holds conventions? -She seems deeply nettled by the sheer brazenness of this behavior, -though to an outsider, this activity might make one wonder -whether hackers should be considered "criminals" at all. -Skateboarders have magazines, and they trespass a lot. -Hot rod people have magazines and they break speed limits -and sometimes kill people. . . . - -I ask her whether it would be any loss to society if phone phreaking -and computer hacking, as hobbies, simply dried up and blew away, -so that nobody ever did it again. - -She seems surprised. "No," she says swiftly. "Maybe a little. . . -in the old days. . .the MIT stuff. . . . But there's a lot of wonderful, -legal stuff you can do with computers now, you don't have to break into -somebody else's just to learn. You don't have that excuse. -You can learn all you like." - -Did you ever hack into a system? I ask. - -The trainees do it at Glynco. Just to demonstrate system vulnerabilities. -She's cool to the notion. Genuinely indifferent. - -"What kind of computer do you have?" - -"A Compaq 286LE," she mutters. - -"What kind do you WISH you had?" - -At this question, the unmistakable light of true hackerdom flares in -Gail Thackeray's eyes. She becomes tense, animated, the words pour out: -"An Amiga 2000 with an IBM card and Mac emulation! The most common hacker -machines are Amigas and Commodores. And Apples." If she had the Amiga, -she enthuses, she could run a whole galaxy of seized computer-evidence disks -on one convenient multifunctional machine. A cheap one, too. Not like the -old Attorney General lab, where they had an ancient CP/M machine, -assorted Amiga flavors and Apple flavors, a couple IBMS, all the -utility software. . .but no Commodores. The workstations down -at the Attorney General's are Wang dedicated word-processors. -Lame machines tied in to an office net--though at least they get -on- line to the Lexis and Westlaw legal data services. - -I don't say anything. I recognize the syndrome, though. -This computer-fever has been running through segments of -our society for years now. It's a strange kind of lust: -K-hunger, Meg-hunger; but it's a shared disease; -it can kill parties dead, as conversation spirals into -the deepest and most deviant recesses of software releases -and expensive peripherals. . . . The mark of the hacker beast. -I have it too. The whole "electronic community," whatever the hell -that is, has it. Gail Thackeray has it. Gail Thackeray is a hacker cop. -My immediate reaction is a strong rush of indignant pity: -WHY DOESN'T SOMEBODY BUY THIS WOMAN HER AMIGA?! -It's not like she's asking for a Cray X-MP -supercomputer mainframe; an Amiga's a sweet little -cookie-box thing. We're losing zillions in organized fraud; -prosecuting and defending a single hacker case in court can cost -a hundred grand easy. How come nobody can come up with four lousy grand -so this woman can do her job? For a hundred grand we could buy every -computer cop in America an Amiga. There aren't that many of 'em. - -Computers. The lust, the hunger, for computers. -The loyalty they inspire, the intense sense of possessiveness. -The culture they have bred. I myself am sitting in downtown Phoenix, -Arizona because it suddenly occurred to me that the police might-- -just MIGHT--come and take away my computer. The prospect of this, -the mere IMPLIED THREAT, was unbearable. It literally changed my life. -It was changing the lives of many others. Eventually it would change -everybody's life. - -Gail Thackeray was one of the top computer-crime people in America. -And I was just some novelist, and yet I had a better computer than hers. -PRACTICALLY EVERYBODY I KNEW had a better computer than Gail Thackeray -and her feeble laptop 286. It was like sending the sheriff in to clean -up Dodge City and arming her with a slingshot cut from an old rubber tire. - -But then again, you don't need a howitzer to enforce the law. -You can do a lot just with a badge. With a badge alone, -you can basically wreak havoc, take a terrible vengeance on wrongdoers. -Ninety percent of "computer crime investigation" is just "crime investigation:" -names, places, dossiers, modus operandi, search warrants, victims, -complainants, informants. . . . - -What will computer crime look like in ten years? Will it get better? -Did "Sundevil" send 'em reeling back in confusion? - -It'll be like it is now, only worse, she tells me with perfect conviction. -Still there in the background, ticking along, changing with the times: -the criminal underworld. It'll be like drugs are. Like our problems -with alcohol. All the cops and laws in the world never solved our problems -with alcohol. If there's something people want, a certain percentage -of them are just going to take it. Fifteen percent of the populace -will never steal. Fifteen percent will steal most anything not nailed down. -The battle is for the hearts and minds of the remaining seventy percent. - -And criminals catch on fast. If there's not "too steep a learning curve"-- -if it doesn't require a baffling amount of expertise and practice-- -then criminals are often some of the first through the gate of a -new technology. Especially if it helps them to hide. -They have tons of cash, criminals. The new communications tech-- -like pagers, cellular phones, faxes, Federal Express--were pioneered -by rich corporate people, and by criminals. In the early years -of pagers and beepers, dope dealers were so enthralled this technology -that owing a beeper was practically prima facie evidence of cocaine dealing. -CB radio exploded when the speed limit hit 55 and breaking the highway law -became a national pastime. Dope dealers send cash by Federal Express, -despite, or perhaps BECAUSE OF, the warnings in FedEx offices that tell you -never to try this. Fed Ex uses X-rays and dogs on their mail, -to stop drug shipments. That doesn't work very well. - -Drug dealers went wild over cellular phones. -There are simple methods of faking ID on cellular phones, -making the location of the call mobile, free of charge, -and effectively untraceable. Now victimized cellular -companies routinely bring in vast toll-lists of calls -to Colombia and Pakistan. - -Judge Greene's fragmentation of the phone company -is driving law enforcement nuts. Four thousand -telecommunications companies. Fraud skyrocketing. -Every temptation in the world available with a phone -and a credit card number. Criminals untraceable. -A galaxy of "new neat rotten things to do." - -If there were one thing Thackeray would like to have, -it would be an effective legal end-run through this new -fragmentation minefield. - -It would be a new form of electronic search warrant, -an "electronic letter of marque" to be issued by a judge. -It would create a new category of "electronic emergency." -Like a wiretap, its use would be rare, but it would cut -across state lines and force swift cooperation from all concerned. -Cellular, phone, laser, computer network, PBXes, AT&T, Baby Bells, -long-distance entrepreneurs, packet radio. Some document, -some mighty court-order, that could slice through four thousand -separate forms of corporate red-tape, and get her at once to -the source of calls, the source of email threats and viruses, -the sources of bomb threats, kidnapping threats. "From now on," -she says, "the Lindbergh baby will always die." - -Something that would make the Net sit still, if only for a moment. -Something that would get her up to speed. Seven league boots. -That's what she really needs. "Those guys move in nanoseconds -and I'm on the Pony Express." - -And then, too, there's the coming international angle. -Electronic crime has never been easy to localize, -to tie to a physical jurisdiction. And phone-phreaks -and hackers loathe boundaries, they jump them whenever they can. -The English. The Dutch. And the Germans, especially the ubiquitous -Chaos Computer Club. The Australians. They've all learned phone-phreaking -from America. It's a growth mischief industry. The multinational -networks are global, but governments and the police simply aren't. -Neither are the laws. Or the legal frameworks for citizen protection. - -One language is global, though--English. Phone phreaks speak English; -it's their native tongue even if they're Germans. English may have started -in England but now it's the Net language; it might as well be called "CNNese." - -Asians just aren't much into phone phreaking. They're the world masters -at organized software piracy. The French aren't into phone-phreaking either. -The French are into computerized industrial espionage. - -In the old days of the MIT righteous hackerdom, crashing systems -didn't hurt anybody. Not all that much, anyway. Not permanently. -Now the players are more venal. Now the consequences are worse. -Hacking will begin killing people soon. Already there are methods -of stacking calls onto 911 systems, annoying the police, and possibly -causing the death of some poor soul calling in with a genuine emergency. -Hackers in Amtrak computers, or air-traffic control computers, will kill -somebody someday. Maybe a lot of people. Gail Thackeray expects it. - -And the viruses are getting nastier. The "Scud" virus is the latest one out. -It wipes hard-disks. - -According to Thackeray, the idea that phone-phreaks are Robin Hoods is a fraud. -They don't deserve this repute. Basically, they pick on the weak. AT&T now -protects itself with the fearsome ANI (Automatic Number Identification) -trace capability. When AT&T wised up and tightened security generally, -the phreaks drifted into the Baby Bells. The Baby Bells lashed out in 1989 -and 1990, so the phreaks switched to smaller long-distance entrepreneurs. -Today, they are moving into locally owned PBXes and voice-mail systems, -which are full of security holes, dreadfully easy to hack. These victims -aren't the moneybags Sheriff of Nottingham or Bad King John, but small groups -of innocent people who find it hard to protect themselves, and who really -suffer from these depredations. Phone phreaks pick on the weak. They do it -for power. If it were legal, they wouldn't do it. They don't want service, -or knowledge, they want the thrill of power-tripping. There's plenty of -knowledge or service around if you're willing to pay. Phone phreaks don't pay, -they steal. It's because it is illegal that it feels like power, -that it gratifies their vanity. - -I leave Gail Thackeray with a handshake at the door of her office building-- -a vast International-Style office building downtown. The Sheriff's office -is renting part of it. I get the vague impression that quite a lot of the -building is empty--real estate crash. - -In a Phoenix sports apparel store, in a downtown mall, I meet -the "Sun Devil" himself. He is the cartoon mascot of -Arizona State University, whose football stadium, "Sundevil," -is near the local Secret Service HQ--hence the name Operation Sundevil. -The Sun Devil himself is named "Sparky." Sparky the Sun Devil is maroon -and bright yellow, the school colors. Sparky brandishes a three-tined -yellow pitchfork. He has a small mustache, pointed ears, a barbed tail, -and is dashing forward jabbing the air with the pitchfork, -with an expression of devilish glee. - -Phoenix was the home of Operation Sundevil. The Legion of Doom -ran a hacker bulletin board called "The Phoenix Project." -An Australian hacker named "Phoenix" once burrowed through -the Internet to attack Cliff Stoll, then bragged and boasted -about it to The New York Times. This net of coincidence -is both odd and meaningless. - -The headquarters of the Arizona Attorney General, Gail Thackeray's -former workplace, is on 1275 Washington Avenue. Many of the downtown -streets in Phoenix are named after prominent American presidents: -Washington, Jefferson, Madison. . . . - -After dark, all the employees go home to their suburbs. -Washington, Jefferson and Madison--what would be the -Phoenix inner city, if there were an inner city in this -sprawling automobile-bred town--become the haunts -of transients and derelicts. The homeless. The sidewalks -along Washington are lined with orange trees. -Ripe fallen fruit lies scattered like croquet balls -on the sidewalks and gutters. No one seems to be eating them. -I try a fresh one. It tastes unbearably bitter. - -The Attorney General's office, built in 1981 during the -Babbitt administration, is a long low two-story building -of white cement and wall-sized sheets of curtain-glass. -Behind each glass wall is a lawyer's office, quite open -and visible to anyone strolling by. Across the street -is a dour government building labelled simply ECONOMIC SECURITY, -something that has not been in great supply in the American -Southwest lately. - -The offices are about twelve feet square. They feature -tall wooden cases full of red-spined lawbooks; -Wang computer monitors; telephones; Post-it notes galore. -Also framed law diplomas and a general excess of bad -Western landscape art. Ansel Adams photos are a big favorite, -perhaps to compensate for the dismal specter of the parking lot, -two acres of striped black asphalt, which features gravel landscaping -and some sickly-looking barrel cacti. - -It has grown dark. Gail Thackeray has told me that the people -who work late here, are afraid of muggings in the parking lot. -It seems cruelly ironic that a woman tracing electronic racketeers -across the interstate labyrinth of Cyberspace should fear an assault -by a homeless derelict in the parking lot of her own workplace. - -Perhaps this is less than coincidence. Perhaps these two seemingly -disparate worlds are somehow generating one another. The poor and -disenfranchised take to the streets, while the rich and computer-equipped, -safe in their bedrooms, chatter over their modems. Quite often the derelicts -kick the glass out and break in to the lawyers' offices, if they see something -they need or want badly enough. - -I cross the parking lot to the street behind the Attorney General's office. -A pair of young tramps are bedding down on flattened sheets of cardboard, -under an alcove stretching over the sidewalk. One tramp wears a -glitter-covered T-shirt reading "CALIFORNIA" in Coca-Cola cursive. -His nose and cheeks look chafed and swollen; they glisten with -what seems to be Vaseline. The other tramp has a ragged long-sleeved -shirt and lank brown hair parted in the middle. They both wear blue jeans -coated in grime. They are both drunk. - -"You guys crash here a lot?" I ask them. - -They look at me warily. I am wearing black jeans, a black pinstriped -suit jacket and a black silk tie. I have odd shoes and a funny haircut. - -"It's our first time here," says the red-nosed tramp unconvincingly. -There is a lot of cardboard stacked here. More than any two people could use. - -"We usually stay at the Vinnie's down the street," says the brown-haired tramp, -puffing a Marlboro with a meditative air, as he sprawls with his head on -a blue nylon backpack. "The Saint Vincent's." - -"You know who works in that building over there?" I ask, pointing. - -The brown-haired tramp shrugs. "Some kind of attorneys, it says." - -We urge one another to take it easy. I give them five bucks. - -A block down the street I meet a vigorous workman who is wheeling along -some kind of industrial trolley; it has what appears to be a tank of -propane on it. - -We make eye contact. We nod politely. I walk past him. "Hey! -Excuse me sir!" he says. - -"Yes?" I say, stopping and turning. - -"Have you seen," the guy says rapidly, "a black guy, about 6'7", -scars on both his cheeks like this--" he gestures-- "wears a -black baseball cap on backwards, wandering around here anyplace?" - -"Sounds like I don't much WANT to meet him," I say. - -"He took my wallet," says my new acquaintance. -"Took it this morning. Y'know, some people would be -SCARED of a guy like that. But I'm not scared. -I'm from Chicago. I'm gonna hunt him down. -We do things like that in Chicago." - -"Yeah?" - -"I went to the cops and now he's got an APB out on his ass," -he says with satisfaction. "You run into him, you let me know." - -"Okay," I say. "What is your name, sir?" - -"Stanley. . . ." - -"And how can I reach you?" - -"Oh," Stanley says, in the same rapid voice, -"you don't have to reach, uh, me. -You can just call the cops. Go straight to the cops." -He reaches into a pocket and pulls out a greasy piece of pasteboard. -"See, here's my report on him." - -I look. The "report," the size of an index card, is labelled PRO-ACT: -Phoenix Residents Opposing Active Crime Threat. . . . or is it -Organized Against Crime Threat? In the darkening street it's hard -to read. Some kind of vigilante group? Neighborhood watch? -I feel very puzzled. - -"Are you a police officer, sir?" - -He smiles, seems very pleased by the question. - -"No," he says. - -"But you are a `Phoenix Resident?'" - -"Would you believe a homeless person," Stanley says. - -"Really? But what's with the. . . ." For the first time I take a close look -at Stanley's trolley. It's a rubber-wheeled thing of industrial metal, -but the device I had mistaken for a tank of propane is in fact a water-cooler. -Stanley also has an Army duffel-bag, stuffed tight as a sausage with clothing -or perhaps a tent, and, at the base of his trolley, a cardboard box and a -battered leather briefcase. - -"I see," I say, quite at a loss. For the first time I notice that Stanley -has a wallet. He has not lost his wallet at all. It is in his back pocket -and chained to his belt. It's not a new wallet. It seems to have seen -a lot of wear. - -"Well, you know how it is, brother," says Stanley. -Now that I know that he is homeless--A POSSIBLE -THREAT--my entire perception of him has changed -in an instant. His speech, which once seemed just -bright and enthusiastic, now seems to have a -dangerous tang of mania. "I have to do this!" -he assures me. "Track this guy down. . . . -It's a thing I do. . . you know. . .to keep myself together!" -He smiles, nods, lifts his trolley by its decaying rubber handgrips. - -"Gotta work together, y'know," Stanley booms, his face alight -with cheerfulness, "the police can't do everything!" -The gentlemen I met in my stroll in downtown Phoenix -are the only computer illiterates in this book. -To regard them as irrelevant, however, would be a grave mistake. - -As computerization spreads across society, the populace at large -is subjected to wave after wave of future shock. But, as a -necessary converse, the "computer community" itself is subjected -to wave after wave of incoming computer illiterates. -How will those currently enjoying America's digital bounty regard, -and treat, all this teeming refuse yearning to breathe free? -Will the electronic frontier be another Land of Opportunity-- -or an armed and monitored enclave, where the disenfranchised -snuggle on their cardboard at the locked doors of our houses of justice? - -Some people just don't get along with computers. They can't read. -They can't type. They just don't have it in their heads to master -arcane instructions in wirebound manuals. Somewhere, the process -of computerization of the populace will reach a limit. Some people-- -quite decent people maybe, who might have thrived in any other situation-- -will be left irretrievably outside the bounds. What's to be done with -these people, in the bright new shiny electroworld? How will they -be regarded, by the mouse-whizzing masters of cyberspace? With contempt? -Indifference? Fear? - -In retrospect, it astonishes me to realize how quickly poor Stanley -became a perceived threat. Surprise and fear are closely allied feelings. -And the world of computing is full of surprises. - -I met one character in the streets of Phoenix whose role in this book -is supremely and directly relevant. That personage was Stanley's giant -thieving scarred phantom. This phantasm is everywhere in this book. -He is the specter haunting cyberspace. - -Sometimes he's a maniac vandal ready to smash the phone system -for no sane reason at all. Sometimes he's a fascist fed, -coldly programming his mighty mainframes to destroy our Bill of Rights. -Sometimes he's a telco bureaucrat, covertly conspiring to register all modems -in the service of an Orwellian surveillance regime. Mostly, though, -this fearsome phantom is a "hacker." He's strange, he doesn't belong, -he's not authorized, he doesn't smell right, he's not keeping his proper place, -he's not one of us. The focus of fear is the hacker, for much the same -reasons that Stanley's fancied assailant is black. - -Stanley's demon can't go away, because he doesn't exist. -Despite singleminded and tremendous effort, he can't be arrested, -sued, jailed, or fired. The only constructive way to do ANYTHING -about him is to learn more about Stanley himself. This learning process -may be repellent, it may be ugly, it may involve grave elements of paranoiac -confusion, but it's necessary. Knowing Stanley requires something more -than class-crossing condescension. It requires more than steely -legal objectivity. It requires human compassion and sympathy. - -To know Stanley is to know his demon. If you know the other guy's demon, -then maybe you'll come to know some of your own. You'll be able to -separate reality from illusion. And then you won't do your cause, -and yourself, more harm than good. Like poor damned Stanley from Chicago did. - -# - -The Federal Computer Investigations Committee (FCIC) is the most important -and influential organization in the realm of American computer-crime. -Since the police of other countries have largely taken their computer-crime -cues from American methods, the FCIC might well be called the most important -computer crime group in the world. - -It is also, by federal standards, an organization of great unorthodoxy. -State and local investigators mix with federal agents. Lawyers, -financial auditors and computer-security programmers trade notes -with street cops. Industry vendors and telco security people show up -to explain their gadgetry and plead for protection and justice. -Private investigators, think-tank experts and industry pundits throw in -their two cents' worth. The FCIC is the antithesis of a formal bureaucracy. - -Members of the FCIC are obscurely proud of this fact; they recognize their -group as aberrant, but are entirely convinced that this, for them, -outright WEIRD behavior is nevertheless ABSOLUTELY NECESSARY -to get their jobs done. - -FCIC regulars --from the Secret Service, the FBI, the IRS, -the Department of Labor, the offices of federal attorneys, -state police, the Air Force, from military intelligence-- -often attend meetings, held hither and thither across the country, -at their own expense. The FCIC doesn't get grants. It doesn't -charge membership fees. It doesn't have a boss. It has no headquarters-- -just a mail drop in Washington DC, at the Fraud Division of the Secret Service. -It doesn't have a budget. It doesn't have schedules. It meets three times -a year--sort of. Sometimes it issues publications, but the FCIC -has no regular publisher, no treasurer, not even a secretary. -There are no minutes of FCIC meetings. Non-federal people are considered -"non-voting members," but there's not much in the way of elections. -There are no badges, lapel pins or certificates of membership. -Everyone is on a first-name basis. There are about forty of them. -Nobody knows how many, exactly. People come, people go-- -sometimes people "go" formally but still hang around anyway. -Nobody has ever exactly figured out what "membership" of this -"Committee" actually entails. - -Strange as this may seem to some, to anyone familiar with the social world -of computing, the "organization" of the FCIC is very recognizable. - -For years now, economists and management theorists have speculated -that the tidal wave of the information revolution would destroy rigid, -pyramidal bureaucracies, where everything is top-down and -centrally controlled. Highly trained "employees" would take on -much greater autonomy, being self-starting, and self-motivating, -moving from place to place, task to task, with great speed and fluidity. -"Ad-hocracy" would rule, with groups of people spontaneously knitting -together across organizational lines, tackling the problem at hand, -applying intense computer-aided expertise to it, and then vanishing -whence they came. - -This is more or less what has actually happened in the world of -federal computer investigation. With the conspicuous exception -of the phone companies, which are after all over a hundred years old, -practically EVERY organization that plays any important role in this book -functions just like the FCIC. The Chicago Task Force, the Arizona -Racketeering Unit, the Legion of Doom, the Phrack crowd, the -Electronic Frontier Foundation--they ALL look and act like "tiger teams" -or "user's groups." They are all electronic ad-hocracies leaping up -spontaneously to attempt to meet a need. - -Some are police. Some are, by strict definition, criminals. -Some are political interest-groups. But every single group -has that same quality of apparent spontaneity--"Hey, gang! -My uncle's got a barn--let's put on a show!" - -Every one of these groups is embarrassed by this "amateurism," -and, for the sake of their public image in a world of non-computer people, -they all attempt to look as stern and formal and impressive as possible. -These electronic frontier-dwellers resemble groups of nineteenth-century -pioneers hankering after the respectability of statehood. -There are however, two crucial differences in the historical experience -of these "pioneers" of the nineteeth and twenty-first centuries. - -First, powerful information technology DOES play into the hands of small, -fluid, loosely organized groups. There have always been "pioneers," -"hobbyists," "amateurs," "dilettantes," "volunteers," "movements," -"users' groups" and "blue-ribbon panels of experts" around. -But a group of this kind--when technically equipped to ship -huge amounts of specialized information, at lightning speed, -to its members, to government, and to the press--is simply -a different kind of animal. It's like the difference between -an eel and an electric eel. - -The second crucial change is that American society is currently -in a state approaching permanent technological revolution. -In the world of computers particularly, it is practically impossible -to EVER stop being a "pioneer," unless you either drop dead or -deliberately jump off the bus. The scene has never slowed down -enough to become well-institutionalized. And after twenty, thirty, -forty years the "computer revolution" continues to spread, -to permeate new corners of society. Anything that really works -is already obsolete. - -If you spend your entire working life as a "pioneer," the word "pioneer" -begins to lose its meaning. Your way of life looks less and less like -an introduction to something else" more stable and organized, -and more and more like JUST THE WAY THINGS ARE. A "permanent revolution" -is really a contradiction in terms. If "turmoil" lasts long enough, -it simply becomes A NEW KIND OF SOCIETY--still the same game of history, -but new players, new rules. - -Apply this to the world of late twentieth-century law enforcement, -and the implications are novel and puzzling indeed. Any bureaucratic -rulebook you write about computer-crime will be flawed when you write it, -and almost an antique by the time it sees print. The fluidity and fast -reactions of the FCIC give them a great advantage in this regard, -which explains their success. Even with the best will in the world -(which it does not, in fact, possess) it is impossible for an organization -the size of the U.S. Federal Bureau of Investigation to get up to speed -on the theory and practice of computer crime. If they tried to train all -their agents to do this, it would be SUICIDAL, as they would NEVER BE ABLE -TO DO ANYTHING ELSE. - -The FBI does try to train its agents in the basics of electronic crime, -at their base in Quantico, Virginia. And the Secret Service, along with -many other law enforcement groups, runs quite successful and well-attended -training courses on wire fraud, business crime, and computer intrusion -at the Federal Law Enforcement Training Center (FLETC, pronounced "fletsy") -in Glynco, Georgia. But the best efforts of these bureaucracies does not -remove the absolute need for a "cutting-edge mess" like the FCIC. - -For you see--the members of FCIC ARE the trainers of the rest -of law enforcement. Practically and literally speaking, -they are the Glynco computer-crime faculty by another name. -If the FCIC went over a cliff on a bus, the U.S. law enforcement -community would be rendered deaf dumb and blind in the world -of computer crime, and would swiftly feel a desperate need -to reinvent them. And this is no time to go starting from scratch. - -On June 11, 1991, I once again arrived in Phoenix, Arizona, -for the latest meeting of the Federal Computer Investigations Committee. -This was more or less the twentieth meeting of this stellar group. -The count was uncertain, since nobody could figure out whether to -include the meetings of "the Colluquy," which is what the FCIC -was called in the mid-1980s before it had even managed to obtain -the dignity of its own acronym. - -Since my last visit to Arizona, in May, the local AzScam bribery scandal -had resolved itself in a general muddle of humiliation. The Phoenix chief -of police, whose agents had videotaped nine state legislators up to no good, -had resigned his office in a tussle with the Phoenix city council over -the propriety of his undercover operations. - -The Phoenix Chief could now join Gail Thackeray and eleven of her closest -associates in the shared experience of politically motivated unemployment. -As of June, resignations were still continuing at the Arizona Attorney -General's office, which could be interpreted as either a New Broom -Sweeping Clean or a Night of the Long Knives Part II, depending on -your point of view. - -The meeting of FCIC was held at the Scottsdale Hilton Resort. -Scottsdale is a wealthy suburb of Phoenix, known as "Scottsdull" -to scoffing local trendies, but well-equipped with posh shopping-malls -and manicured lawns, while conspicuously undersupplied with homeless derelicts. -The Scottsdale Hilton Resort was a sprawling hotel in postmodern -crypto-Southwestern style. It featured a "mission bell tower" -plated in turquoise tile and vaguely resembling a Saudi minaret. - -Inside it was all barbarically striped Santa Fe Style decor. -There was a health spa downstairs and a large oddly-shaped -pool in the patio. A poolside umbrella-stand offered Ben and Jerry's -politically correct Peace Pops. - -I registered as a member of FCIC, attaining a handy discount rate, -then went in search of the Feds. Sure enough, at the back of the -hotel grounds came the unmistakable sound of Gail Thackeray -holding forth. - -Since I had also attended the Computers Freedom and Privacy conference -(about which more later), this was the second time I had seen Thackeray -in a group of her law enforcement colleagues. Once again I was struck -by how simply pleased they seemed to see her. It was natural that she'd -get SOME attention, as Gail was one of two women in a group of some thirty men; -but there was a lot more to it than that. - -Gail Thackeray personifies the social glue of the FCIC. They could give -a damn about her losing her job with the Attorney General. They were sorry -about it, of course, but hell, they'd all lost jobs. If they were the kind -of guys who liked steady boring jobs, they would never have gotten into -computer work in the first place. - -I wandered into her circle and was immediately introduced to five strangers. -The conditions of my visit at FCIC were reviewed. I would not quote -anyone directly. I would not tie opinions expressed to the agencies -of the attendees. I would not (a purely hypothetical example) -report the conversation of a guy from the Secret Service talking -quite civilly to a guy from the FBI, as these two agencies NEVER -talk to each other, and the IRS (also present, also hypothetical) -NEVER TALKS TO ANYBODY. - -Worse yet, I was forbidden to attend the first conference. And I didn't. -I have no idea what the FCIC was up to behind closed doors that afternoon. -I rather suspect that they were engaging in a frank and thorough confession -of their errors, goof-ups and blunders, as this has been a feature of every -FCIC meeting since their legendary Memphis beer-bust of 1986. Perhaps the -single greatest attraction of FCIC is that it is a place where you can go, -let your hair down, and completely level with people who actually comprehend -what you are talking about. Not only do they understand you, but they -REALLY PAY ATTENTION, they are GRATEFUL FOR YOUR INSIGHTS, and they -FORGIVE YOU, which in nine cases out of ten is something even your -boss can't do, because as soon as you start talking "ROM," "BBS," -or "T-1 trunk," his eyes glaze over. - -I had nothing much to do that afternoon. The FCIC were beavering away -in their conference room. Doors were firmly closed, windows too dark -to peer through. I wondered what a real hacker, a computer intruder, -would do at a meeting like this. - -The answer came at once. He would "trash" the place. Not reduce the place -to trash in some orgy of vandalism; that's not the use of the term in the -hacker milieu. No, he would quietly EMPTY THE TRASH BASKETS and silently -raid any valuable data indiscreetly thrown away. - -Journalists have been known to do this. (Journalists hunting information -have been known to do almost every single unethical thing that hackers -have ever done. They also throw in a few awful techniques all their own.) -The legality of `trashing' is somewhat dubious but it is not in fact -flagrantly illegal. It was, however, absurd to contemplate trashing the FCIC. -These people knew all about trashing. I wouldn't last fifteen seconds. - -The idea sounded interesting, though. I'd been hearing a lot about -the practice lately. On the spur of the moment, I decided I would try -trashing the office ACROSS THE HALL from the FCIC, an area which had -nothing to do with the investigators. - -The office was tiny; six chairs, a table. . . . Nevertheless, it was open, -so I dug around in its plastic trash can. - -To my utter astonishment, I came up with the torn scraps of a SPRINT -long-distance phone bill. More digging produced a bank statement -and the scraps of a hand-written letter, along with gum, cigarette ashes, -candy wrappers and a day-old-issue of USA TODAY. - -The trash went back in its receptacle while the scraps of data went into -my travel bag. I detoured through the hotel souvenir shop for some -Scotch tape and went up to my room. - -Coincidence or not, it was quite true. Some poor soul had, in fact, -thrown a SPRINT bill into the hotel's trash. Date May 1991, -total amount due: $252.36. Not a business phone, either, -but a residential bill, in the name of someone called Evelyn -(not her real name). Evelyn's records showed a ## PAST DUE BILL ##! -Here was her nine-digit account ID. Here was a stern computer-printed warning: - -"TREAT YOUR FONCARD AS YOU WOULD ANY CREDIT CARD. TO SECURE AGAINST FRAUD, -NEVER GIVE YOUR FONCARD NUMBER OVER THE PHONE UNLESS YOU INITIATED THE CALL. -IF YOU RECEIVE SUSPICIOUS CALLS PLEASE NOTIFY CUSTOMER SERVICE IMMEDIATELY!" - -I examined my watch. Still plenty of time left for the FCIC to carry on. -I sorted out the scraps of Evelyn's SPRINT bill and re-assembled them with -fresh Scotch tape. Here was her ten-digit FONCARD number. Didn't seem -to have the ID number necessary to cause real fraud trouble. - -I did, however, have Evelyn's home phone number. And the phone numbers -for a whole crowd of Evelyn's long-distance friends and acquaintances. -In San Diego, Folsom, Redondo, Las Vegas, La Jolla, Topeka, and Northampton -Massachusetts. Even somebody in Australia! - -I examined other documents. Here was a bank statement. It was Evelyn's -IRA account down at a bank in San Mateo California (total balance $1877.20). -Here was a charge-card bill for $382.64. She was paying it off bit by bit. - -Driven by motives that were completely unethical and prurient, -I now examined the handwritten notes. They had been torn fairly -thoroughly, so much so that it took me almost an entire five minutes -to reassemble them. - -They were drafts of a love letter. They had been written on -the lined stationery of Evelyn's employer, a biomedical company. -Probably written at work when she should have been doing something else. - -"Dear Bob," (not his real name) "I guess in everyone's life there comes -a time when hard decisions have to be made, and this is a difficult one -for me--very upsetting. Since you haven't called me, and I don't understand -why, I can only surmise it's because you don't want to. I thought I would -have heard from you Friday. I did have a few unusual problems with my phone -and possibly you tried, I hope so. - -"Robert, you asked me to `let go'. . . ." - -The first note ended. UNUSUAL PROBLEMS WITH HER PHONE? -I looked swiftly at the next note. - -"Bob, not hearing from you for the whole weekend has left me very perplexed. . . ." - -Next draft. - -"Dear Bob, there is so much I don't understand right now, and I wish I did. -I wish I could talk to you, but for some unknown reason you have elected not -to call--this is so difficult for me to understand. . . ." - -She tried again. - -"Bob, Since I have always held you in such high esteem, I had every hope that -we could remain good friends, but now one essential ingredient is missing-- -respect. Your ability to discard people when their purpose is served is -appalling to me. The kindest thing you could do for me now is to leave me -alone. You are no longer welcome in my heart or home. . . ." - -Try again. - -"Bob, I wrote a very factual note to you to say how much respect I had lost -for you, by the way you treat people, me in particular, so uncaring and cold. -The kindest thing you can do for me is to leave me alone entirely, -as you are no longer welcome in my heart or home. I would appreciate it -if you could retire your debt to me as soon as possible--I wish no link -to you in any way. Sincerely, Evelyn." - -Good heavens, I thought, the bastard actually owes her money! -I turned to the next page. - -"Bob: very simple. GOODBYE! No more mind games--no more fascination-- -no more coldness--no more respect for you! It's over--Finis. Evie" - -There were two versions of the final brushoff letter, but they read about -the same. Maybe she hadn't sent it. The final item in my illicit and -shameful booty was an envelope addressed to "Bob" at his home address, -but it had no stamp on it and it hadn't been mailed. - -Maybe she'd just been blowing off steam because her rascal boyfriend -had neglected to call her one weekend. Big deal. Maybe they'd kissed -and made up, maybe she and Bob were down at Pop's Chocolate Shop now, -sharing a malted. Sure. - -Easy to find out. All I had to do was call Evelyn up. With a half-clever -story and enough brass-plated gall I could probably trick the truth out of her. -Phone-phreaks and hackers deceive people over the phone all the time. -It's called "social engineering." Social engineering is a very common practice -in the underground, and almost magically effective. Human beings are almost -always the weakest link in computer security. The simplest way to learn -Things You Are Not Meant To Know is simply to call up and exploit the -knowledgeable people. With social engineering, you use the bits of specialized -knowledge you already have as a key, to manipulate people into believing -that you are legitimate. You can then coax, flatter, or frighten them into -revealing almost anything you want to know. Deceiving people (especially -over the phone) is easy and fun. Exploiting their gullibility is very -gratifying; it makes you feel very superior to them. - -If I'd been a malicious hacker on a trashing raid, I would now have Evelyn -very much in my power. Given all this inside data, it wouldn't take much -effort at all to invent a convincing lie. If I were ruthless enough, -and jaded enough, and clever enough, this momentary indiscretion of hers-- -maybe committed in tears, who knows--could cause her a whole world of -confusion and grief. - -I didn't even have to have a MALICIOUS motive. Maybe I'd be "on her side," -and call up Bob instead, and anonymously threaten to break both his kneecaps -if he didn't take Evelyn out for a steak dinner pronto. It was still -profoundly NONE OF MY BUSINESS. To have gotten this knowledge at all -was a sordid act and to use it would be to inflict a sordid injury. - -To do all these awful things would require exactly zero high-tech expertise. -All it would take was the willingness to do it and a certain amount -of bent imagination. - -I went back downstairs. The hard-working FCIC, who had labored forty-five -minutes over their schedule, were through for the day, and adjourned to the -hotel bar. We all had a beer. - -I had a chat with a guy about "Isis," or rather IACIS, -the International Association of Computer Investigation Specialists. -They're into "computer forensics," the techniques of picking computer- -systems apart without destroying vital evidence. IACIS, currently run -out of Oregon, is comprised of investigators in the U.S., Canada, Taiwan -and Ireland. "Taiwan and Ireland?" I said. Are TAIWAN and IRELAND -really in the forefront of this stuff? Well not exactly, my informant -admitted. They just happen to have been the first ones to have caught -on by word of mouth. Still, the international angle counts, because this -is obviously an international problem. Phone-lines go everywhere. - -There was a Mountie here from the Royal Canadian Mounted Police. -He seemed to be having quite a good time. Nobody had flung this -Canadian out because he might pose a foreign security risk. -These are cyberspace cops. They still worry a lot about "jurisdictions," -but mere geography is the least of their troubles. - -NASA had failed to show. NASA suffers a lot from computer intrusions, -in particular from Australian raiders and a well-trumpeted Chaos -Computer Club case, and in 1990 there was a brief press flurry -when it was revealed that one of NASA's Houston branch-exchanges -had been systematically ripped off by a gang of phone-phreaks. -But the NASA guys had had their funding cut. They were stripping everything. - -Air Force OSI, its Office of Special Investigations, is the ONLY federal -entity dedicated full-time to computer security. They'd been expected -to show up in force, but some of them had cancelled--a Pentagon budget pinch. - -As the empties piled up, the guys began joshing around and telling war-stories. -"These are cops," Thackeray said tolerantly. "If they're not talking shop -they talk about women and beer." - -I heard the story about the guy who, asked for "a copy" of a computer disk, -PHOTOCOPIED THE LABEL ON IT. He put the floppy disk onto the glass plate -of a photocopier. The blast of static when the copier worked completely -erased all the real information on the disk. - -Some other poor souls threw a whole bag of confiscated diskettes -into the squad-car trunk next to the police radio. The powerful radio -signal blasted them, too. - -We heard a bit about Dave Geneson, the first computer prosecutor, -a mainframe-runner in Dade County, turned lawyer. Dave Geneson -was one guy who had hit the ground running, a signal virtue -in making the transition to computer-crime. It was generally -agreed that it was easier to learn the world of computers first, -then police or prosecutorial work. You could take certain computer -people and train 'em to successful police work--but of course they -had to have the COP MENTALITY. They had to have street smarts. -Patience. Persistence. And discretion. You've got to make sure -they're not hot-shots, show-offs, "cowboys." - -Most of the folks in the bar had backgrounds in military intelligence, -or drugs, or homicide. It was rudely opined that "military intelligence" -was a contradiction in terms, while even the grisly world of homicide -was considered cleaner than drug enforcement. One guy had been 'way -undercover doing dope-work in Europe for four years straight. -"I'm almost recovered now," he said deadpan, with the acid black humor -that is pure cop. "Hey, now I can say FUCKER without putting MOTHER -in front of it." - -"In the cop world," another guy said earnestly, "everything is good and bad, -black and white. In the computer world everything is gray." - -One guy--a founder of the FCIC, who'd been with the group -since it was just the Colluquy--described his own introduction -to the field. He'd been a Washington DC homicide guy called in -on a "hacker" case. From the word "hacker," he naturally assumed -he was on the trail of a knife-wielding marauder, and went to the -computer center expecting blood and a body. When he finally figured -out what was happening there (after loudly demanding, in vain, -that the programmers "speak English"), he called headquarters -and told them he was clueless about computers. They told him nobody -else knew diddly either, and to get the hell back to work. - -So, he said, he had proceeded by comparisons. By analogy. By metaphor. -"Somebody broke in to your computer, huh?" Breaking and entering; -I can understand that. How'd he get in? "Over the phone-lines." -Harassing phone-calls, I can understand that! What we need here -is a tap and a trace! - -It worked. It was better than nothing. And it worked a lot faster -when he got hold of another cop who'd done something similar. -And then the two of them got another, and another, and pretty soon -the Colluquy was a happening thing. It helped a lot that everybody -seemed to know Carlton Fitzpatrick, the data-processing trainer in Glynco. - -The ice broke big-time in Memphis in '86. The Colluquy had attracted -a bunch of new guys--Secret Service, FBI, military, other feds, heavy guys. -Nobody wanted to tell anybody anything. They suspected that if word got back -to the home office they'd all be fired. They passed an uncomfortably -guarded afternoon. - -The formalities got them nowhere. But after the formal session was over, -the organizers brought in a case of beer. As soon as the participants -knocked it off with the bureaucratic ranks and turf-fighting, everything -changed. "I bared my soul," one veteran reminisced proudly. By nightfall -they were building pyramids of empty beer-cans and doing everything -but composing a team fight song. - -FCIC were not the only computer-crime people around. There was DATTA -(District Attorneys' Technology Theft Association), though they mostly -specialized in chip theft, intellectual property, and black-market cases. -There was HTCIA (High Tech Computer Investigators Association), -also out in Silicon Valley, a year older than FCIC and featuring -brilliant people like Donald Ingraham. There was LEETAC -(Law Enforcement Electronic Technology Assistance Committee) -in Florida, and computer-crime units in Illinois and Maryland -and Texas and Ohio and Colorado and Pennsylvania. But these were -local groups. FCIC were the first to really network nationally -and on a federal level. - -FCIC people live on the phone lines. Not on bulletin board systems-- -they know very well what boards are, and they know that boards aren't secure. -Everyone in the FCIC has a voice-phone bill like you wouldn't believe. -FCIC people have been tight with the telco people for a long time. -Telephone cyberspace is their native habitat. - -FCIC has three basic sub-tribes: the trainers, the security people, -and the investigators. That's why it's called an "Investigations -Committee" with no mention of the term "computer-crime"--the dreaded -"C-word." FCIC, officially, is "an association of agencies rather -than individuals;" unofficially, this field is small enough that -the influence of individuals and individual expertise is paramount. -Attendance is by invitation only, and most everyone in FCIC considers -himself a prophet without honor in his own house. - -Again and again I heard this, with different terms but identical -sentiments. "I'd been sitting in the wilderness talking to myself." -"I was totally isolated." "I was desperate." "FCIC is the best -thing there is about computer crime in America." "FCIC is what -really works." "This is where you hear real people telling you -what's really happening out there, not just lawyers picking nits." -"We taught each other everything we knew." - -The sincerity of these statements convinces me that this is true. -FCIC is the real thing and it is invaluable. It's also very sharply -at odds with the rest of the traditions and power structure -in American law enforcement. There probably hasn't been anything -around as loose and go-getting as the FCIC since the start of the -U.S. Secret Service in the 1860s. FCIC people are living like -twenty-first-century people in a twentieth-century environment, -and while there's a great deal to be said for that, there's also -a great deal to be said against it, and those against it happen -to control the budgets. - -I listened to two FCIC guys from Jersey compare life histories. -One of them had been a biker in a fairly heavy-duty gang in the 1960s. -"Oh, did you know so-and-so?" said the other guy from Jersey. -"Big guy, heavyset?" - -"Yeah, I knew him." - -"Yeah, he was one of ours. He was our plant in the gang." - -"Really? Wow! Yeah, I knew him. Helluva guy." - -Thackeray reminisced at length about being tear-gassed blind -in the November 1969 antiwar protests in Washington Circle, -covering them for her college paper. "Oh yeah, I was there," -said another cop. "Glad to hear that tear gas hit somethin'. -Haw haw haw." He'd been so blind himself, he confessed, -that later that day he'd arrested a small tree. - -FCIC are an odd group, sifted out by coincidence and necessity, -and turned into a new kind of cop. There are a lot of specialized -cops in the world--your bunco guys, your drug guys, your tax guys, -but the only group that matches FCIC for sheer isolation are probably -the child-pornography people. Because they both deal with conspirators -who are desperate to exchange forbidden data and also desperate to hide; -and because nobody else in law enforcement even wants to hear about it. - -FCIC people tend to change jobs a lot. They tend not to get the equipment -and training they want and need. And they tend to get sued quite often. - -As the night wore on and a band set up in the bar, the talk grew darker. -Nothing ever gets done in government, someone opined, until there's -a DISASTER. Computing disasters are awful, but there's no denying -that they greatly help the credibility of FCIC people. The Internet Worm, -for instance. "For years we'd been warning about that--but it's nothing -compared to what's coming." They expect horrors, these people. -They know that nothing will really get done until there is a horror. - -# - -Next day we heard an extensive briefing from a guy who'd been a computer cop, -gotten into hot water with an Arizona city council, and now installed -computer networks for a living (at a considerable rise in pay). -He talked about pulling fiber-optic networks apart. - -Even a single computer, with enough peripherals, is a literal -"network"--a bunch of machines all cabled together, generally -with a complexity that puts stereo units to shame. FCIC people -invent and publicize methods of seizing computers and maintaining -their evidence. Simple things, sometimes, but vital rules of thumb -for street cops, who nowadays often stumble across a busy computer -in the midst of a drug investigation or a white-collar bust. -For instance: Photograph the system before you touch it. -Label the ends of all the cables before you detach anything. -"Park" the heads on the disk drives before you move them. -Get the diskettes. Don't put the diskettes in magnetic fields. -Don't write on diskettes with ballpoint pens. Get the manuals. -Get the printouts. Get the handwritten notes. Copy data before -you look at it, and then examine the copy instead of the original. - -Now our lecturer distributed copied diagrams of a typical LAN -or "Local Area Network", which happened to be out of Connecticut. -ONE HUNDRED AND FIFTY-NINE desktop computers, each with its own -peripherals. Three "file servers." Five "star couplers" -each with thirty-two ports. One sixteen-port coupler -off in the corner office. All these machines talking to each other, -distributing electronic mail, distributing software, distributing, -quite possibly, criminal evidence. All linked by high-capacity -fiber-optic cable. A bad guy--cops talk a about "bad guys" ---might be lurking on PC #47 lot or #123 and distributing -his ill doings onto some dupe's "personal" machine in -another office--or another floor--or, quite possibly, -two or three miles away! Or, conceivably, the evidence might -be "data-striped"--split up into meaningless slivers stored, -one by one, on a whole crowd of different disk drives. - -The lecturer challenged us for solutions. I for one was utterly clueless. -As far as I could figure, the Cossacks were at the gate; there were probably -more disks in this single building than were seized during the entirety -of Operation Sundevil. - -"Inside informant," somebody said. Right. There's always the human angle, -something easy to forget when contemplating the arcane recesses of high -technology. Cops are skilled at getting people to talk, and computer people, -given a chair and some sustained attention, will talk about their computers -till their throats go raw. There's a case on record of a single question-- -"How'd you do it?"--eliciting a forty-five-minute videotaped confession -from a computer criminal who not only completely incriminated himself -but drew helpful diagrams. - -Computer people talk. Hackers BRAG. Phone-phreaks -talk PATHOLOGICALLY--why else are they stealing phone-codes, -if not to natter for ten hours straight to their friends -on an opposite seaboard? Computer-literate people do -in fact possess an arsenal of nifty gadgets and techniques -that would allow them to conceal all kinds of exotic skullduggery, -and if they could only SHUT UP about it, they could probably -get away with all manner of amazing information-crimes. -But that's just not how it works--or at least, -that's not how it's worked SO FAR. - -Most every phone-phreak ever busted has swiftly implicated his mentors, -his disciples, and his friends. Most every white-collar computer-criminal, -smugly convinced that his clever scheme is bulletproof, swiftly learns -otherwise when, for the first time in his life, an actual no-kidding -policeman leans over, grabs the front of his shirt, looks him right -in the eye and says: "All right, ASSHOLE--you and me are going downtown!" -All the hardware in the world will not insulate your nerves from -these actual real-life sensations of terror and guilt. - -Cops know ways to get from point A to point Z without thumbing -through every letter in some smart-ass bad-guy's alphabet. -Cops know how to cut to the chase. Cops know a lot of things -other people don't know. - -Hackers know a lot of things other people don't know, too. -Hackers know, for instance, how to sneak into your computer -through the phone-lines. But cops can show up RIGHT ON YOUR DOORSTEP -and carry off YOU and your computer in separate steel boxes. -A cop interested in hackers can grab them and grill them. -A hacker interested in cops has to depend on hearsay, -underground legends, and what cops are willing to publicly reveal. -And the Secret Service didn't get named "the SECRET Service" -because they blab a lot. - -Some people, our lecturer informed us, were under the mistaken -impression that it was "impossible" to tap a fiber-optic line. -Well, he announced, he and his son had just whipped up a -fiber-optic tap in his workshop at home. He passed it around -the audience, along with a circuit-covered LAN plug-in card -so we'd all recognize one if we saw it on a case. We all had a look. - -The tap was a classic "Goofy Prototype"--a thumb-length rounded -metal cylinder with a pair of plastic brackets on it. -From one end dangled three thin black cables, each of which ended -in a tiny black plastic cap. When you plucked the safety-cap -off the end of a cable, you could see the glass fiber-- -no thicker than a pinhole. - -Our lecturer informed us that the metal cylinder was a -"wavelength division multiplexer." Apparently, what one did -was to cut the fiber-optic cable, insert two of the legs into -the cut to complete the network again, and then read any passing data -on the line by hooking up the third leg to some kind of monitor. -Sounded simple enough. I wondered why nobody had thought of it before. -I also wondered whether this guy's son back at the workshop had any -teenage friends. - -We had a break. The guy sitting next to me was wearing a giveaway -baseball cap advertising the Uzi submachine gun. We had a desultory chat -about the merits of Uzis. Long a favorite of the Secret Service, -it seems Uzis went out of fashion with the advent of the Persian Gulf War, -our Arab allies taking some offense at Americans toting Israeli weapons. -Besides, I was informed by another expert, Uzis jam. The equivalent weapon -of choice today is the Heckler & Koch, manufactured in Germany. - -The guy with the Uzi cap was a forensic photographer. He also did a lot -of photographic surveillance work in computer crime cases. He used to, -that is, until the firings in Phoenix. He was now a private investigator and, -with his wife, ran a photography salon specializing in weddings and portrait -photos. At--one must repeat--a considerable rise in income. - -He was still FCIC. If you were FCIC, and you needed to talk -to an expert about forensic photography, well, there he was, -willing and able. If he hadn't shown up, people would have missed him. - -Our lecturer had raised the point that preliminary investigation -of a computer system is vital before any seizure is undertaken. -It's vital to understand how many machines are in there, what kinds -there are, what kind of operating system they use, how many people -use them, where the actual data itself is stored. To simply barge into -an office demanding "all the computers" is a recipe for swift disaster. - -This entails some discreet inquiries beforehand. In fact, what it -entails is basically undercover work. An intelligence operation. -SPYING, not to put too fine a point on it. - -In a chat after the lecture, I asked an attendee whether "trashing" might work. - -I received a swift briefing on the theory and practice of "trash covers." -Police "trash covers," like "mail covers" or like wiretaps, require the -agreement of a judge. This obtained, the "trashing" work of cops is just -like that of hackers, only more so and much better organized. So much so, -I was informed, that mobsters in Phoenix make extensive use of locked -garbage cans picked up by a specialty high-security trash company. - -In one case, a tiger team of Arizona cops had trashed a local residence -for four months. Every week they showed up on the municipal garbage truck, -disguised as garbagemen, and carried the contents of the suspect cans off -to a shade tree, where they combed through the garbage--a messy task, -especially considering that one of the occupants was undergoing -kidney dialysis. All useful documents were cleaned, dried and examined. -A discarded typewriter-ribbon was an especially valuable source of data, -as its long one-strike ribbon of film contained the contents of every -letter mailed out of the house. The letters were neatly retyped by -a police secretary equipped with a large desk-mounted magnifying glass. - -There is something weirdly disquieting about the whole subject of -"trashing"-- an unsuspected and indeed rather disgusting mode of -deep personal vulnerability. Things that we pass by every day, -that we take utterly for granted, can be exploited with so little work. -Once discovered, the knowledge of these vulnerabilities tend to spread. - -Take the lowly subject of MANHOLE COVERS. The humble manhole cover -reproduces many of the dilemmas of computer-security in miniature. -Manhole covers are, of course, technological artifacts, access-points -to our buried urban infrastructure. To the vast majority of us, -manhole covers are invisible. They are also vulnerable. For many years now, -the Secret Service has made a point of caulking manhole covers along all routes -of the Presidential motorcade. This is, of course, to deter terrorists from -leaping out of underground ambush or, more likely, planting remote-control -car-smashing bombs beneath the street. - -Lately, manhole covers have seen more and more criminal exploitation, -especially in New York City. Recently, a telco in New York City -discovered that a cable television service had been sneaking into -telco manholes and installing cable service alongside the phone-lines-- -WITHOUT PAYING ROYALTIES. New York companies have also suffered a -general plague of (a) underground copper cable theft; (b) dumping of garbage, -including toxic waste, and (c) hasty dumping of murder victims. - -Industry complaints reached the ears of an innovative New England -industrial-security company, and the result was a new product known -as "the Intimidator," a thick titanium-steel bolt with a precisely machined -head that requires a special device to unscrew. All these "keys" have registered -serial numbers kept on file with the manufacturer. There are now some -thousands of these "Intimidator" bolts being sunk into American pavements -wherever our President passes, like some macabre parody of strewn roses. -They are also spreading as fast as steel dandelions around US military bases -and many centers of private industry. - -Quite likely it has never occurred to you to peer under a manhole cover, -perhaps climb down and walk around down there with a flashlight, just to see -what it's like. Formally speaking, this might be trespassing, but if you -didn't hurt anything, and didn't make an absolute habit of it, nobody would -really care. The freedom to sneak under manholes was likely a freedom -you never intended to exercise. - -You now are rather less likely to have that freedom at all. -You may never even have missed it until you read about it here, -but if you're in New York City it's gone, and elsewhere it's likely going. -This is one of the things that crime, and the reaction to -crime, does to us. - -The tenor of the meeting now changed as the Electronic Frontier Foundation -arrived. The EFF, whose personnel and history will be examined in detail -in the next chapter, are a pioneering civil liberties group who arose in -direct response to the Hacker Crackdown of 1990. - -Now Mitchell Kapor, the Foundation's president, and Michael Godwin, -its chief attorney, were confronting federal law enforcement MANO A MANO -for the first time ever. Ever alert to the manifold uses of publicity, -Mitch Kapor and Mike Godwin had brought their own journalist in tow: -Robert Draper, from Austin, whose recent well-received book about -ROLLING STONE magazine was still on the stands. Draper was on assignment -for TEXAS MONTHLY. - -The Steve Jackson/EFF civil lawsuit against the Chicago Computer Fraud -and Abuse Task Force was a matter of considerable regional interest in Texas. -There were now two Austinite journalists here on the case. In fact, -counting Godwin (a former Austinite and former journalist) there were -three of us. Lunch was like Old Home Week. - -Later, I took Draper up to my hotel room. We had a long frank talk -about the case, networking earnestly like a miniature freelance-journo -version of the FCIC: privately confessing the numerous blunders -of journalists covering the story, and trying hard to figure out -who was who and what the hell was really going on out there. -I showed Draper everything I had dug out of the Hilton trashcan. -We pondered the ethics of "trashing" for a while, and agreed -that they were dismal. We also agreed that finding a SPRINT -bill on your first time out was a heck of a coincidence. - -First I'd "trashed"--and now, mere hours later, I'd bragged to someone else. -Having entered the lifestyle of hackerdom, I was now, unsurprisingly, -following its logic. Having discovered something remarkable through -a surreptitious action, I of course HAD to "brag," and to drag the passing -Draper into my iniquities. I felt I needed a witness. Otherwise nobody -would have believed what I'd discovered. . . . - -Back at the meeting, Thackeray cordially, if rather tentatively, -introduced Kapor and Godwin to her colleagues. Papers were distributed. -Kapor took center stage. The brilliant Bostonian high-tech entrepreneur, -normally the hawk in his own administration and quite an effective -public speaker, seemed visibly nervous, and frankly admitted as much. -He began by saying he consided computer-intrusion to be morally wrong, -and that the EFF was not a "hacker defense fund," despite what had appeared -in print. Kapor chatted a bit about the basic motivations of his group, -emphasizing their good faith and willingness to listen and seek common ground -with law enforcement--when, er, possible. - -Then, at Godwin's urging, Kapor suddenly remarked that EFF's own Internet -machine had been "hacked" recently, and that EFF did not consider -this incident amusing. - -After this surprising confession, things began to loosen up -quite rapidly. Soon Kapor was fielding questions, parrying objections, -challenging definitions, and juggling paradigms with something akin -to his usual gusto. - -Kapor seemed to score quite an effect with his shrewd and skeptical analysis -of the merits of telco "Caller-ID" services. (On this topic, FCIC and EFF -have never been at loggerheads, and have no particular established earthworks -to defend.) Caller-ID has generally been promoted as a privacy service -for consumers, a presentation Kapor described as a "smokescreen," -the real point of Caller-ID being to ALLOW CORPORATE CUSTOMERS TO BUILD -EXTENSIVE COMMERCIAL DATABASES ON EVERYBODY WHO PHONES OR FAXES THEM. -Clearly, few people in the room had considered this possibility, -except perhaps for two late-arrivals from US WEST RBOC security, -who chuckled nervously. - -Mike Godwin then made an extensive presentation on -"Civil Liberties Implications of Computer Searches and Seizures." -Now, at last, we were getting to the real nitty-gritty here, -real political horse-trading. The audience listened with close -attention, angry mutters rising occasionally: "He's trying to -teach us our jobs!" "We've been thinking about this for years! -We think about these issues every day!" "If I didn't seize the works, -I'd be sued by the guy's victims!" "I'm violating the law if I leave -ten thousand disks full of illegal PIRATED SOFTWARE and STOLEN CODES!" -"It's our job to make sure people don't trash the Constitution-- -we're the DEFENDERS of the Constitution!" "We seize stuff when -we know it will be forfeited anyway as restitution for the victim!" - -"If it's forfeitable, then don't get a search warrant, get a -forfeiture warrant," Godwin suggested coolly. He further remarked -that most suspects in computer crime don't WANT to see their computers -vanish out the door, headed God knew where, for who knows how long. -They might not mind a search, even an extensive search, but they want -their machines searched on-site. - -"Are they gonna feed us?" somebody asked sourly. - -"How about if you take copies of the data?" Godwin parried. - -"That'll never stand up in court." - -"Okay, you make copies, give THEM the copies, and take the originals." - -Hmmm. - -Godwin championed bulletin-board systems as repositories of First Amendment -protected free speech. He complained that federal computer-crime training -manuals gave boards a bad press, suggesting that they are hotbeds of crime -haunted by pedophiles and crooks, whereas the vast majority of the nation's -thousands of boards are completely innocuous, and nowhere near so -romantically suspicious. - -People who run boards violently resent it when their systems are seized, -and their dozens (or hundreds) of users look on in abject horror. -Their rights of free expression are cut short. Their right to associate -with other people is infringed. And their privacy is violated as their -private electronic mail becomes police property. - -Not a soul spoke up to defend the practice of seizing boards. -The issue passed in chastened silence. Legal principles aside-- -(and those principles cannot be settled without laws passed or -court precedents)--seizing bulletin boards has become public-relations -poison for American computer police. - -And anyway, it's not entirely necessary. If you're a cop, you can get 'most -everything you need from a pirate board, just by using an inside informant. -Plenty of vigilantes--well, CONCERNED CITIZENS--will inform police the moment -they see a pirate board hit their area (and will tell the police all about it, -in such technical detail, actually, that you kinda wish they'd shut up). -They will happily supply police with extensive downloads or printouts. -It's IMPOSSIBLE to keep this fluid electronic information out of the -hands of police. - -Some people in the electronic community become enraged at the prospect -of cops "monitoring" bulletin boards. This does have touchy aspects, -as Secret Service people in particular examine bulletin boards with -some regularity. But to expect electronic police to be deaf dumb -and blind in regard to this particular medium rather flies in the face -of common sense. Police watch television, listen to radio, read newspapers -and magazines; why should the new medium of boards be different? -Cops can exercise the same access to electronic information -as everybody else. As we have seen, quite a few computer -police maintain THEIR OWN bulletin boards, including anti-hacker -"sting" boards, which have generally proven quite effective. - -As a final clincher, their Mountie friends in Canada (and colleagues -in Ireland and Taiwan) don't have First Amendment or American -constitutional restrictions, but they do have phone lines, -and can call any bulletin board in America whenever they please. -The same technological determinants that play into the hands of hackers, -phone phreaks and software pirates can play into the hands of police. -"Technological determinants" don't have ANY human allegiances. -They're not black or white, or Establishment or Underground, -or pro-or-anti anything. - -Godwin complained at length about what he called "the Clever Hobbyist -hypothesis" --the assumption that the "hacker" you're busting is clearly -a technical genius, and must therefore by searched with extreme thoroughness. -So: from the law's point of view, why risk missing anything? Take the works. -Take the guy's computer. Take his books. Take his notebooks. -Take the electronic drafts of his love letters. Take his Walkman. -Take his wife's computer. Take his dad's computer. Take his kid -sister's computer. Take his employer's computer. Take his compact disks-- -they MIGHT be CD-ROM disks, cunningly disguised as pop music. -Take his laser printer--he might have hidden something vital in the -printer's 5meg of memory. Take his software manuals and hardware -documentation. Take his science-fiction novels and his simulation- -gaming books. Take his Nintendo Game-Boy and his Pac-Man arcade game. -Take his answering machine, take his telephone out of the wall. -Take anything remotely suspicious. - -Godwin pointed out that most "hackers" are not, in fact, clever -genius hobbyists. Quite a few are crooks and grifters who don't -have much in the way of technical sophistication; just some rule-of-thumb -rip-off techniques. The same goes for most fifteen-year-olds who've -downloaded a code-scanning program from a pirate board. There's no -real need to seize everything in sight. It doesn't require an entire -computer system and ten thousand disks to prove a case in court. - -What if the computer is the instrumentality of a crime? someone demanded. - -Godwin admitted quietly that the doctrine of seizing the instrumentality -of a crime was pretty well established in the American legal system. - -The meeting broke up. Godwin and Kapor had to leave. Kapor was testifying -next morning before the Massachusetts Department Of Public Utility, -about ISDN narrowband wide-area networking. - -As soon as they were gone, Thackeray seemed elated. -She had taken a great risk with this. Her colleagues had not, -in fact, torn Kapor and Godwin's heads off. She was very proud of them, -and told them so. - -"Did you hear what Godwin said about INSTRUMENTALITY OF A CRIME?" -she exulted, to nobody in particular. "Wow, that means -MITCH ISN'T GOING TO SUE ME." - -# - -America's computer police are an interesting group. -As a social phenomenon they are far more interesting, -and far more important, than teenage phone phreaks -and computer hackers. First, they're older and wiser; -not dizzy hobbyists with leaky morals, but seasoned adult -professionals with all the responsibilities of public service. -And, unlike hackers, they possess not merely TECHNICAL -power alone, but heavy-duty legal and social authority. - -And, very interestingly, they are just as much at -sea in cyberspace as everyone else. They are not -happy about this. Police are authoritarian by nature, -and prefer to obey rules and precedents. (Even those police -who secretly enjoy a fast ride in rough territory will soberly -disclaim any "cowboy" attitude.) But in cyberspace there ARE -no rules and precedents. They are groundbreaking pioneers, -Cyberspace Rangers, whether they like it or not. - -In my opinion, any teenager enthralled by computers, -fascinated by the ins and outs of computer security, -and attracted by the lure of specialized forms of knowledge and power, -would do well to forget all about "hacking" and set his (or her) -sights on becoming a fed. Feds can trump hackers at almost every -single thing hackers do, including gathering intelligence, -undercover disguise, trashing, phone-tapping, building dossiers, -networking, and infiltrating computer systems--CRIMINAL computer systems. -Secret Service agents know more about phreaking, coding and carding -than most phreaks can find out in years, and when it comes to viruses, -break-ins, software bombs and trojan horses, Feds have direct access to red-hot -confidential information that is only vague rumor in the underground. - -And if it's an impressive public rep you're after, there are few people -in the world who can be so chillingly impressive as a well-trained, -well-armed United States Secret Service agent. - -Of course, a few personal sacrifices are necessary in order to obtain -that power and knowledge. First, you'll have the galling discipline -of belonging to a large organization; but the world of computer crime -is still so small, and so amazingly fast-moving, that it will remain -spectacularly fluid for years to come. The second sacrifice is that -you'll have to give up ripping people off. This is not a great loss. -Abstaining from the use of illegal drugs, also necessary, will be a boon -to your health. - -A career in computer security is not a bad choice for a young man -or woman today. The field will almost certainly expand drastically -in years to come. If you are a teenager today, by the time you -become a professional, the pioneers you have read about in this book -will be the grand old men and women of the field, swamped by their many -disciples and successors. Of course, some of them, like William P. Wood -of the 1865 Secret Service, may well be mangled in the whirring machinery -of legal controversy; but by the time you enter the computer-crime field, -it may have stabilized somewhat, while remaining entertainingly challenging. - -But you can't just have a badge. You have to win it. First, there's the -federal law enforcement training. And it's hard--it's a challenge. -A real challenge--not for wimps and rodents. - -Every Secret Service agent must complete gruelling courses at the -Federal Law Enforcement Training Center. (In fact, Secret Service -agents are periodically re-trained during their entire careers.) - -In order to get a glimpse of what this might be like, -I myself travelled to FLETC. - -# - -The Federal Law Enforcement Training Center is a 1500-acre facility -on Georgia's Atlantic coast. It's a milieu of marshgrass, seabirds, -damp, clinging sea-breezes, palmettos, mosquitos, and bats. -Until 1974, it was a Navy Air Base, and still features a working runway, -and some WWII vintage blockhouses and officers' quarters. -The Center has since benefitted by a forty-million-dollar retrofit, -but there's still enough forest and swamp on the facility for the -Border Patrol to put in tracking practice. - -As a town, "Glynco" scarcely exists. The nearest real town is Brunswick, -a few miles down Highway 17, where I stayed at the aptly named Marshview -Holiday Inn. I had Sunday dinner at a seafood restaurant called "Jinright's," -where I feasted on deep-fried alligator tail. This local favorite was -a heaped basket of bite-sized chunks of white, tender, almost fluffy -reptile meat, steaming in a peppered batter crust. Alligator makes -a culinary experience that's hard to forget, especially when liberally -basted with homemade cocktail sauce from a Jinright squeeze-bottle. - -The crowded clientele were tourists, fishermen, local black folks -in their Sunday best, and white Georgian locals who all seemed -to bear an uncanny resemblance to Georgia humorist Lewis Grizzard. - -The 2,400 students from 75 federal agencies who make up the FLETC -population scarcely seem to make a dent in the low-key local scene. -The students look like tourists, and the teachers seem to have taken -on much of the relaxed air of the Deep South. My host was Mr. Carlton -Fitzpatrick, the Program Coordinator of the Financial Fraud Institute. -Carlton Fitzpatrick is a mustached, sinewy, well-tanned Alabama native -somewhere near his late forties, with a fondness for chewing tobacco, -powerful computers, and salty, down-home homilies. We'd met before, -at FCIC in Arizona. - -The Financial Fraud Institute is one of the nine divisions at FLETC. -Besides Financial Fraud, there's Driver & Marine, Firearms, -and Physical Training. These are specialized pursuits. -There are also five general training divisions: Basic Training, -Operations, Enforcement Techniques, Legal Division, and Behavioral Science. - -Somewhere in this curriculum is everything necessary to turn green college -graduates into federal agents. First they're given ID cards. Then they get -the rather miserable-looking blue coveralls known as "smurf suits." -The trainees are assigned a barracks and a cafeteria, and immediately -set on FLETC's bone-grinding physical training routine. Besides the -obligatory daily jogging--(the trainers run up danger flags beside -the track when the humidity rises high enough to threaten heat stroke)-- -here's the Nautilus machines, the martial arts, the survival skills. . . . - -The eighteen federal agencies who maintain on-site academies at FLETC -employ a wide variety of specialized law enforcement units, some of them -rather arcane. There's Border Patrol, IRS Criminal Investigation Division, -Park Service, Fish and Wildlife, Customs, Immigration, Secret Service and -the Treasury's uniformed subdivisions. . . . If you're a federal cop -and you don't work for the FBI, you train at FLETC. This includes people -as apparently obscure as the agents of the Railroad Retirement Board -Inspector General. Or the Tennessee Valley Authority Police, -who are in fact federal police officers, and can and do arrest criminals -on the federal property of the Tennessee Valley Authority. - -And then there are the computer-crime people. All sorts, all backgrounds. -Mr. Fitzpatrick is not jealous of his specialized knowledge. Cops all over, -in every branch of service, may feel a need to learn what he can teach. -Backgrounds don't matter much. Fitzpatrick himself was originally a -Border Patrol veteran, then became a Border Patrol instructor at FLETC. -His Spanish is still fluent--but he found himself strangely fascinated -when the first computers showed up at the Training Center. Fitzpatrick -did have a background in electrical engineering, and though he never -considered himself a computer hacker, he somehow found himself writing -useful little programs for this new and promising gizmo. - -He began looking into the general subject of computers and crime, -reading Donn Parker's books and articles, keeping an ear cocked -for war stories, useful insights from the field, the up-and-coming -people of the local computer-crime and high-technology units. . . . -Soon he got a reputation around FLETC as the resident "computer expert," -and that reputation alone brought him more exposure, more experience-- -until one day he looked around, and sure enough he WAS a federal -computer-crime expert. - -In fact, this unassuming, genial man may be THE federal computer-crime expert. -There are plenty of very good computer people, and plenty of very good -federal investigators, but the area where these worlds of expertise overlap -is very slim. And Carlton Fitzpatrick has been right at the center of that -since 1985, the first year of the Colluquy, a group which owes much to -his influence. - -He seems quite at home in his modest, acoustic-tiled office, -with its Ansel Adams-style Western photographic art, a gold-framed -Senior Instructor Certificate, and a towering bookcase crammed with -three-ring binders with ominous titles such as Datapro Reports on -Information Security and CFCA Telecom Security '90. - -The phone rings every ten minutes; colleagues show up at the door -to chat about new developments in locksmithing or to shake their heads -over the latest dismal developments in the BCCI global banking scandal. - -Carlton Fitzpatrick is a fount of computer-crime war-stories, -related in an acerbic drawl. He tells me the colorful tale -of a hacker caught in California some years back. He'd been -raiding systems, typing code without a detectable break, -for twenty, twenty-four, thirty-six hours straight. Not just -logged on--TYPING. Investigators were baffled. Nobody -could do that. Didn't he have to go to the bathroom? -Was it some kind of automatic keyboard-whacking device -that could actually type code? - -A raid on the suspect's home revealed a situation of astonishing squalor. -The hacker turned out to be a Pakistani computer-science student who had -flunked out of a California university. He'd gone completely underground -as an illegal electronic immigrant, and was selling stolen phone-service -to stay alive. The place was not merely messy and dirty, but in a state -of psychotic disorder. Powered by some weird mix of culture shock, -computer addiction, and amphetamines, the suspect had in fact been sitting -in front of his computer for a day and a half straight, with snacks and -drugs at hand on the edge of his desk and a chamber-pot under his chair. - -Word about stuff like this gets around in the hacker-tracker community. - -Carlton Fitzpatrick takes me for a guided tour by car around the -FLETC grounds. One of our first sights is the biggest indoor -firing range in the world. There are federal trainees in there, -Fitzpatrick assures me politely, blasting away with a wide variety -of automatic weapons: Uzis, Glocks, AK-47s. . . . He's willing to -take me inside. I tell him I'm sure that's really interesting, -but I'd rather see his computers. Carlton Fitzpatrick seems quite -surprised and pleased. I'm apparently the first journalist he's ever -seen who has turned down the shooting gallery in favor of microchips. - -Our next stop is a favorite with touring Congressmen: the three-mile -long FLETC driving range. Here trainees of the Driver & Marine Division -are taught high-speed pursuit skills, setting and breaking road-blocks, -diplomatic security driving for VIP limousines. . . . A favorite FLETC -pastime is to strap a passing Senator into the passenger seat beside a -Driver & Marine trainer, hit a hundred miles an hour, then take it right into -"the skid-pan," a section of greased track where two tons of Detroit iron -can whip and spin like a hockey puck. - -Cars don't fare well at FLETC. First they're rifled again and again -for search practice. Then they do 25,000 miles of high-speed -pursuit training; they get about seventy miles per set -of steel-belted radials. Then it's off to the skid pan, -where sometimes they roll and tumble headlong in the grease. -When they're sufficiently grease-stained, dented, and creaky, -they're sent to the roadblock unit, where they're battered without pity. -And finally then they're sacrificed to the Bureau of Alcohol, -Tobacco and Firearms, whose trainees learn the ins and outs -of car-bomb work by blowing them into smoking wreckage. - -There's a railroad box-car on the FLETC grounds, and a large -grounded boat, and a propless plane; all training-grounds for searches. -The plane sits forlornly on a patch of weedy tarmac next to an eerie -blockhouse known as the "ninja compound," where anti-terrorism specialists -practice hostage rescues. As I gaze on this creepy paragon of modern -low-intensity warfare, my nerves are jangled by a sudden staccato outburst -of automatic weapons fire, somewhere in the woods to my right. -"Nine-millimeter," Fitzpatrick judges calmly. - -Even the eldritch ninja compound pales somewhat compared -to the truly surreal area known as "the raid-houses." -This is a street lined on both sides with nondescript -concrete-block houses with flat pebbled roofs. -They were once officers' quarters. Now they are training grounds. -The first one to our left, Fitzpatrick tells me, has been specially -adapted for computer search-and-seizure practice. Inside it has been -wired for video from top to bottom, with eighteen pan-and-tilt -remotely controlled videocams mounted on walls and in corners. -Every movement of the trainee agent is recorded live by teachers, -for later taped analysis. Wasted movements, hesitations, possibly lethal -tactical mistakes--all are gone over in detail. - -Perhaps the weirdest single aspect of this building is its front door, -scarred and scuffed all along the bottom, from the repeated impact, -day after day, of federal shoe-leather. - -Down at the far end of the row of raid-houses some people are practicing -a murder. We drive by slowly as some very young and rather nervous-looking -federal trainees interview a heavyset bald man on the raid-house lawn. -Dealing with murder takes a lot of practice; first you have to learn -to control your own instinctive disgust and panic, then you have to learn -to control the reactions of a nerve-shredded crowd of civilians, -some of whom may have just lost a loved one, some of whom may be murderers-- -quite possibly both at once. - -A dummy plays the corpse. The roles of the bereaved, the morbidly curious, -and the homicidal are played, for pay, by local Georgians: waitresses, -musicians, most anybody who needs to moonlight and can learn a script. -These people, some of whom are FLETC regulars year after year, -must surely have one of the strangest jobs in the world. - -Something about the scene: "normal" people in a weird situation, -standing around talking in bright Georgia sunshine, unsuccessfully -pretending that something dreadful has gone on, while a dummy lies -inside on faked bloodstains. . . . While behind this weird masquerade, -like a nested set of Russian dolls, are grim future realities of real death, -real violence, real murders of real people, that these young agents -will really investigate, many times during their careers. . . . -Over and over. . . . Will those anticipated murders look like this, -feel like this--not as "real" as these amateur actors are trying to -make it seem, but both as "real," and as numbingly unreal, as watching -fake people standing around on a fake lawn? Something about this scene -unhinges me. It seems nightmarish to me, Kafkaesque. I simply don't -know how to take it; my head is turned around; I don't know whether to laugh, -cry, or just shudder. - -When the tour is over, Carlton Fitzpatrick and I talk about computers. -For the first time cyberspace seems like quite a comfortable place. -It seems very real to me suddenly, a place where I know what I'm talking about, -a place I'm used to. It's real. "Real." Whatever. - -Carlton Fitzpatrick is the only person I've met in cyberspace circles -who is happy with his present equipment. He's got a 5 Meg RAM PC with -a 112 meg hard disk; a 660 meg's on the way. He's got a Compaq 386 desktop, -and a Zenith 386 laptop with 120 meg. Down the hall is a NEC Multi-Sync 2A -with a CD-ROM drive and a 9600 baud modem with four com-lines. -There's a training minicomputer, and a 10-meg local mini just for the Center, -and a lab-full of student PC clones and half-a-dozen Macs or so. -There's a Data General MV 2500 with 8 meg on board and a 370 meg disk. - -Fitzpatrick plans to run a UNIX board on the Data General when he's -finished beta-testing the software for it, which he wrote himself. -It'll have E-mail features, massive files on all manner of computer-crime -and investigation procedures, and will follow the computer-security -specifics of the Department of Defense "Orange Book." He thinks -it will be the biggest BBS in the federal government. - -Will it have Phrack on it? I ask wryly. - -Sure, he tells me. Phrack, TAP, Computer Underground Digest, -all that stuff. With proper disclaimers, of course. - -I ask him if he plans to be the sysop. Running a system that size is very -time-consuming, and Fitzpatrick teaches two three-hour courses every day. - -No, he says seriously, FLETC has to get its money worth out of the instructors. -He thinks he can get a local volunteer to do it, a high-school student. - -He says a bit more, something I think about an Eagle Scout law-enforcement -liaison program, but my mind has rocketed off in disbelief. - -"You're going to put a TEENAGER in charge of a federal security BBS?" -I'm speechless. It hasn't escaped my notice that the FLETC Financial -Fraud Institute is the ULTIMATE hacker-trashing target; there is stuff in here, -stuff of such utter and consummate cool by every standard of the -digital underground. . . . - -I imagine the hackers of my acquaintance, fainting dead-away from -forbidden-knowledge greed-fits, at the mere prospect of cracking -the superultra top-secret computers used to train the Secret Service -in computer-crime. . . . - -"Uhm, Carlton," I babble, "I'm sure he's a really nice kid and all, -but that's a terrible temptation to set in front of somebody who's, -you know, into computers and just starting out. . . ." - -"Yeah," he says, "that did occur to me." For the first time I begin -to suspect that he's pulling my leg. - -He seems proudest when he shows me an ongoing project called JICC, -Joint Intelligence Control Council. It's based on the services provided -by EPIC, the El Paso Intelligence Center, which supplies data and intelligence -to the Drug Enforcement Administration, the Customs Service, the Coast Guard, -and the state police of the four southern border states. Certain EPIC files -can now be accessed by drug-enforcement police of Central America, -South America and the Caribbean, who can also trade information -among themselves. Using a telecom program called "White Hat," -written by two brothers named Lopez from the Dominican Republic, -police can now network internationally on inexpensive PCs. -Carlton Fitzpatrick is teaching a class of drug-war agents -from the Third World, and he's very proud of their progress. -Perhaps soon the sophisticated smuggling networks of the -Medellin Cartel will be matched by a sophisticated computer -network of the Medellin Cartel's sworn enemies. They'll track boats, -track contraband, track the international drug-lords who now leap over -borders with great ease, defeating the police through the clever use -of fragmented national jurisdictions. - -JICC and EPIC must remain beyond the scope of this book. -They seem to me to be very large topics fraught with complications -that I am not fit to judge. I do know, however, that the international, -computer-assisted networking of police, across national boundaries, -is something that Carlton Fitzpatrick considers very important, -a harbinger of a desirable future. I also know that networks -by their nature ignore physical boundaries. And I also know -that where you put communications you put a community, -and that when those communities become self-aware -they will fight to preserve themselves and to expand their influence. -I make no judgements whether this is good or bad. -It's just cyberspace; it's just the way things are. - -I asked Carlton Fitzpatrick what advice he would have for -a twenty-year-old who wanted to shine someday in the world -of electronic law enforcement. - -He told me that the number one rule was simply not to be -scared of computers. You don't need to be an obsessive -"computer weenie," but you mustn't be buffaloed just because -some machine looks fancy. The advantages computers give -smart crooks are matched by the advantages they give smart cops. -Cops in the future will have to enforce the law "with their heads, -not their holsters." Today you can make good cases without ever -leaving your office. In the future, cops who resist the computer -revolution will never get far beyond walking a beat. - -I asked Carlton Fitzpatrick if he had some single message for the public; -some single thing that he would most like the American public to know -about his work. - -He thought about it while. "Yes," he said finally. "TELL me the rules, -and I'll TEACH those rules!" He looked me straight in the eye. -"I do the best that I can." - - - -PART FOUR: THE CIVIL LIBERTARIANS - - -The story of the Hacker Crackdown, as we have followed it thus far, -has been technological, subcultural, criminal and legal. -The story of the Civil Libertarians, though it partakes -of all those other aspects, is profoundly and thoroughly POLITICAL. - -In 1990, the obscure, long-simmering struggle over the ownership -and nature of cyberspace became loudly and irretrievably public. -People from some of the oddest corners of American society suddenly -found themselves public figures. Some of these people found this -situation much more than they had ever bargained for. They backpedalled, -and tried to retreat back to the mandarin obscurity of their cozy -subcultural niches. This was generally to prove a mistake. - -But the civil libertarians seized the day in 1990. They found themselves -organizing, propagandizing, podium-pounding, persuading, touring, -negotiating, posing for publicity photos, submitting to interviews, -squinting in the limelight as they tried a tentative, but growingly -sophisticated, buck-and-wing upon the public stage. - -It's not hard to see why the civil libertarians should have -this competitive advantage. - -The hackers of the digital underground are an hermetic elite. -They find it hard to make any remotely convincing case for -their actions in front of the general public. Actually, -hackers roundly despise the "ignorant" public, and have never -trusted the judgement of "the system." Hackers do propagandize, -but only among themselves, mostly in giddy, badly spelled manifestos -of class warfare, youth rebellion or naive techie utopianism. -Hackers must strut and boast in order to establish and preserve -their underground reputations. But if they speak out too loudly -and publicly, they will break the fragile surface-tension of the underground, -and they will be harrassed or arrested. Over the longer term, -most hackers stumble, get busted, get betrayed, or simply give up. -As a political force, the digital underground is hamstrung. - -The telcos, for their part, are an ivory tower under protracted seige. -They have plenty of money with which to push their calculated public image, -but they waste much energy and goodwill attacking one another with -slanderous and demeaning ad campaigns. The telcos have suffered -at the hands of politicians, and, like hackers, they don't trust -the public's judgement. And this distrust may be well-founded. -Should the general public of the high-tech 1990s come to understand -its own best interests in telecommunications, that might well pose -a grave threat to the specialized technical power and authority -that the telcos have relished for over a century. The telcos do -have strong advantages: loyal employees, specialized expertise, -influence in the halls of power, tactical allies in law enforcement, -and unbelievably vast amounts of money. But politically speaking, they lack -genuine grassroots support; they simply don't seem to have many friends. - -Cops know a lot of things other people don't know. -But cops willingly reveal only those aspects of their -knowledge that they feel will meet their institutional -purposes and further public order. Cops have respect, -they have responsibilities, they have power in the streets -and even power in the home, but cops don't do particularly -well in limelight. When pressed, they will step out in the -public gaze to threaten bad-guys, or to cajole prominent citizens, -or perhaps to sternly lecture the naive and misguided. -But then they go back within their time-honored fortress -of the station-house, the courtroom and the rule-book. - -The electronic civil libertarians, however, have proven to be -born political animals. They seemed to grasp very early on -the postmodern truism that communication is power. Publicity is power. -Soundbites are power. The ability to shove one's issue onto the public -agenda--and KEEP IT THERE--is power. Fame is power. Simple personal -fluency and eloquence can be power, if you can somehow catch the -public's eye and ear. - -The civil libertarians had no monopoly on "technical power"-- -though they all owned computers, most were not particularly -advanced computer experts. They had a good deal of money, -but nowhere near the earthshaking wealth and the galaxy -of resources possessed by telcos or federal agencies. -They had no ability to arrest people. They carried -out no phreak and hacker covert dirty-tricks. - -But they really knew how to network. - -Unlike the other groups in this book, the civil libertarians -have operated very much in the open, more or less right -in the public hurly-burly. They have lectured audiences galore -and talked to countless journalists, and have learned to -refine their spiels. They've kept the cameras clicking, -kept those faxes humming, swapped that email, -run those photocopiers on overtime, licked envelopes -and spent small fortunes on airfare and long-distance. -In an information society, this open, overt, obvious activity -has proven to be a profound advantage. - -In 1990, the civil libertarians of cyberspace assembled -out of nowhere in particular, at warp speed. This "group" -(actually, a networking gaggle of interested parties -which scarcely deserves even that loose term) has almost nothing -in the way of formal organization. Those formal civil libertarian -organizations which did take an interest in cyberspace issues, -mainly the Computer Professionals for Social Responsibility -and the American Civil Liberties Union, were carried along -by events in 1990, and acted mostly as adjuncts, -underwriters or launching-pads. - -The civil libertarians nevertheless enjoyed the greatest success -of any of the groups in the Crackdown of 1990. At this writing, -their future looks rosy and the political initiative is firmly in their hands. -This should be kept in mind as we study the highly unlikely lives -and lifestyles of the people who actually made this happen. - -# - -In June 1989, Apple Computer, Inc., of Cupertino, -California, had a problem. Someone had illicitly copied -a small piece of Apple's proprietary software, software -which controlled an internal chip driving the Macintosh -screen display. This Color QuickDraw source code was -a closely guarded piece of Apple's intellectual property. -Only trusted Apple insiders were supposed to possess it. - -But the "NuPrometheus League" wanted things otherwise. -This person (or persons) made several illicit copies -of this source code, perhaps as many as two dozen. -He (or she, or they) then put those illicit floppy disks -into envelopes and mailed them to people all over America: -people in the computer industry who were associated with, -but not directly employed by, Apple Computer. - -The NuPrometheus caper was a complex, highly ideological, -and very hacker-like crime. Prometheus, it will be recalled, -stole the fire of the Gods and gave this potent gift to the -general ranks of downtrodden mankind. A similar god-in-the-manger -attitude was implied for the corporate elite of Apple Computer, -while the "Nu" Prometheus had himself cast in the role of rebel demigod. -The illicitly copied data was given away for free. - -The new Prometheus, whoever he was, escaped the -fate of the ancient Greek Prometheus, who was chained -to a rock for centuries by the vengeful gods while an eagle -tore and ate his liver. On the other hand, NuPrometheus -chickened out somewhat by comparison with his role model. -The small chunk of Color QuickDraw code he had filched -and replicated was more or less useless to Apple's -industrial rivals (or, in fact, to anyone else). -Instead of giving fire to mankind, it was more as if -NuPrometheus had photocopied the schematics for part of a Bic lighter. -The act was not a genuine work of industrial espionage. -It was best interpreted as a symbolic, deliberate slap -in the face for the Apple corporate heirarchy. - -Apple's internal struggles were well-known in the industry. Apple's founders, -Jobs and Wozniak, had both taken their leave long since. Their raucous core -of senior employees had been a barnstorming crew of 1960s Californians, -many of them markedly less than happy with the new button-down multimillion -dollar regime at Apple. Many of the programmers and developers who had -invented the Macintosh model in the early 1980s had also taken their leave of -the company. It was they, not the current masters of Apple's corporate fate, -who had invented the stolen Color QuickDraw code. The NuPrometheus stunt -was well-calculated to wound company morale. - -Apple called the FBI. The Bureau takes an interest in high-profile -intellectual-property theft cases, industrial espionage and theft -of trade secrets. These were likely the right people to call, -and rumor has it that the entities responsible were in fact discovered -by the FBI, and then quietly squelched by Apple management. NuPrometheus -was never publicly charged with a crime, or prosecuted, or jailed. -But there were no further illicit releases of Macintosh internal software. -Eventually the painful issue of NuPrometheus was allowed to fade. - -In the meantime, however, a large number of puzzled bystanders -found themselves entertaining surprise guests from the FBI. - -One of these people was John Perry Barlow. Barlow is a most unusual man, -difficult to describe in conventional terms. He is perhaps best known as -a songwriter for the Grateful Dead, for he composed lyrics for -"Hell in a Bucket," "Picasso Moon," "Mexicali Blues," "I Need a Miracle," -and many more; he has been writing for the band since 1970. - -Before we tackle the vexing question as to why a rock lyricist -should be interviewed by the FBI in a computer-crime case, -it might be well to say a word or two about the Grateful Dead. -The Grateful Dead are perhaps the most successful and long-lasting -of the numerous cultural emanations from the Haight-Ashbury district -of San Francisco, in the glory days of Movement politics and -lysergic transcendance. The Grateful Dead are a nexus, a veritable -whirlwind, of applique decals, psychedelic vans, tie-dyed T-shirts, -earth-color denim, frenzied dancing and open and unashamed drug use. -The symbols, and the realities, of Californian freak power surround -the Grateful Dead like knotted macrame. - -The Grateful Dead and their thousands of Deadhead devotees -are radical Bohemians. This much is widely understood. -Exactly what this implies in the 1990s is rather more problematic. - -The Grateful Dead are among the world's most popular -and wealthy entertainers: number 20, according to Forbes magazine, -right between M.C. Hammer and Sean Connery. In 1990, this jeans-clad -group of purported raffish outcasts earned seventeen million dollars. -They have been earning sums much along this line for quite some time now. - -And while the Dead are not investment bankers or three-piece-suit -tax specialists--they are, in point of fact, hippie musicians-- -this money has not been squandered in senseless Bohemian excess. -The Dead have been quietly active for many years, funding various -worthy activities in their extensive and widespread cultural community. - -The Grateful Dead are not conventional players in the American -power establishment. They nevertheless are something of a force -to be reckoned with. They have a lot of money and a lot of friends -in many places, both likely and unlikely. - -The Dead may be known for back-to-the-earth environmentalist rhetoric, -but this hardly makes them anti-technological Luddites. On the contrary, -like most rock musicians, the Grateful Dead have spent their entire adult -lives in the company of complex electronic equipment. They have funds to burn -on any sophisticated tool and toy that might happen to catch their fancy. -And their fancy is quite extensive. - -The Deadhead community boasts any number of recording engineers, -lighting experts, rock video mavens, electronic technicians -of all descriptions. And the drift goes both ways. Steve Wozniak, -Apple's co-founder, used to throw rock festivals. Silicon Valley rocks out. - -These are the 1990s, not the 1960s. Today, for a surprising number of people -all over America, the supposed dividing line between Bohemian and technician -simply no longer exists. People of this sort may have a set of windchimes -and a dog with a knotted kerchief 'round its neck, but they're also quite -likely to own a multimegabyte Macintosh running MIDI synthesizer software -and trippy fractal simulations. These days, even Timothy Leary himself, -prophet of LSD, does virtual-reality computer-graphics demos in -his lecture tours. - -John Perry Barlow is not a member of the Grateful Dead. He is, however, -a ranking Deadhead. - -Barlow describes himself as a "techno-crank." A vague term like -"social activist" might not be far from the mark, either. -But Barlow might be better described as a "poet"--if one keeps in mind -Percy Shelley's archaic definition of poets as "unacknowledged legislators -of the world." - -Barlow once made a stab at acknowledged legislator status. In 1987, -he narrowly missed the Republican nomination for a seat in the -Wyoming State Senate. Barlow is a Wyoming native, the third-generation -scion of a well-to-do cattle-ranching family. He is in his early forties, -married and the father of three daughters. - -Barlow is not much troubled by other people's narrow notions of consistency. -In the late 1980s, this Republican rock lyricist cattle rancher sold his ranch -and became a computer telecommunications devotee. - -The free-spirited Barlow made this transition with ease. He genuinely -enjoyed computers. With a beep of his modem, he leapt from small-town -Pinedale, Wyoming, into electronic contact with a large and lively crowd -of bright, inventive, technological sophisticates from all over the world. -Barlow found the social milieu of computing attractive: its fast-lane pace, -its blue-sky rhetoric, its open-endedness. Barlow began dabbling in -computer journalism, with marked success, as he was a quick study, -and both shrewd and eloquent. He frequently travelled to San Francisco -to network with Deadhead friends. There Barlow made extensive contacts -throughout the Californian computer community, including friendships -among the wilder spirits at Apple. - -In May 1990, Barlow received a visit from a local Wyoming agent of the FBI. -The NuPrometheus case had reached Wyoming. - -Barlow was troubled to find himself under investigation in an -area of his interests once quite free of federal attention. -He had to struggle to explain the very nature of computer-crime -to a headscratching local FBI man who specialized in cattle-rustling. -Barlow, chatting helpfully and demonstrating the wonders of his modem -to the puzzled fed, was alarmed to find all "hackers" generally under -FBI suspicion as an evil influence in the electronic community. -The FBI, in pursuit of a hacker called "NuPrometheus," were tracing -attendees of a suspect group called the Hackers Conference. - -The Hackers Conference, which had been started in 1984, was a -yearly Californian meeting of digital pioneers and enthusiasts. -The hackers of the Hackers Conference had little if anything to do -with the hackers of the digital underground. On the contrary, -the hackers of this conference were mostly well-to-do Californian -high-tech CEOs, consultants, journalists and entrepreneurs. -(This group of hackers were the exact sort of "hackers" -most likely to react with militant fury at any criminal -degradation of the term "hacker.") - -Barlow, though he was not arrested or accused of a crime, -and though his computer had certainly not gone out the door, -was very troubled by this anomaly. He carried the word to the Well. - -Like the Hackers Conference, "the Well" was an emanation of the -Point Foundation. Point Foundation, the inspiration of a wealthy -Californian 60s radical named Stewart Brand, was to be a major -launch-pad of the civil libertarian effort. - -Point Foundation's cultural efforts, like those of their fellow Bay Area -Californians the Grateful Dead, were multifaceted and multitudinous. -Rigid ideological consistency had never been a strong suit of the -Whole Earth Catalog. This Point publication had enjoyed a strong -vogue during the late 60s and early 70s, when it offered hundreds -of practical (and not so practical) tips on communitarian living, -environmentalism, and getting back-to-the-land. The Whole Earth Catalog, -and its sequels, sold two and half million copies and won a -National Book Award. - -With the slow collapse of American radical dissent, the Whole Earth Catalog -had slipped to a more modest corner of the cultural radar; but in its -magazine incarnation, CoEvolution Quarterly, the Point Foundation -continued to offer a magpie potpourri of "access to tools and ideas." - -CoEvolution Quarterly, which started in 1974, was never a widely -popular magazine. Despite periodic outbreaks of millenarian fervor, -CoEvolution Quarterly failed to revolutionize Western civilization -and replace leaden centuries of history with bright new Californian paradigms. -Instead, this propaganda arm of Point Foundation cakewalked a fine line between -impressive brilliance and New Age flakiness. CoEvolution Quarterly carried -no advertising, cost a lot, and came out on cheap newsprint with modest -black-and-white graphics. It was poorly distributed, and spread mostly -by subscription and word of mouth. - -It could not seem to grow beyond 30,000 subscribers. -And yet--it never seemed to shrink much, either. -Year in, year out, decade in, decade out, some strange -demographic minority accreted to support the magazine. -The enthusiastic readership did not seem to have much -in the way of coherent politics or ideals. It was sometimes -hard to understand what held them together (if the often bitter -debate in the letter-columns could be described as "togetherness"). - -But if the magazine did not flourish, it was resilient; it got by. -Then, in 1984, the birth-year of the Macintosh computer, -CoEvolution Quarterly suddenly hit the rapids. Point Foundation -had discovered the computer revolution. Out came the Whole Earth -Software Catalog of 1984, arousing headscratching doubts among -the tie-dyed faithful, and rabid enthusiasm among the nascent -"cyberpunk" milieu, present company included. Point Foundation -started its yearly Hackers Conference, and began to take an -extensive interest in the strange new possibilities of -digital counterculture. CoEvolution Quarterlyfolded its teepee, -replaced by Whole Earth Software Review and eventually by Whole Earth -Review (the magazine's present incarnation, currently under -the editorship of virtual-reality maven Howard Rheingold). - -1985 saw the birth of the "WELL"--the "Whole Earth 'Lectronic Link." -The Well was Point Foundation's bulletin board system. - -As boards went, the Well was an anomaly from the beginning, -and remained one. It was local to San Francisco. -It was huge, with multiple phonelines and enormous files -of commentary. Its complex UNIX-based software might be -most charitably described as "user-opaque." It was run on -a mainframe out of the rambling offices of a non-profit -cultural foundation in Sausalito. And it was crammed with -fans of the Grateful Dead. - -Though the Well was peopled by chattering hipsters of the Bay Area -counterculture, it was by no means a "digital underground" board. -Teenagers were fairly scarce; most Well users (known as "Wellbeings") -were thirty- and forty-something Baby Boomers. They tended to work -in the information industry: hardware, software, telecommunications, -media, entertainment. Librarians, academics, and journalists were -especially common on the Well, attracted by Point Foundation's -open-handed distribution of "tools and ideas." - -There were no anarchy files on the Well, scarcely a -dropped hint about access codes or credit-card theft. -No one used handles. Vicious "flame-wars" were held to -a comparatively civilized rumble. Debates were sometimes sharp, -but no Wellbeing ever claimed that a rival had disconnected his phone, -trashed his house, or posted his credit card numbers. - -The Well grew slowly as the 1980s advanced. It charged a modest sum -for access and storage, and lost money for years--but not enough to hamper -the Point Foundation, which was nonprofit anyway. By 1990, the Well -had about five thousand users. These users wandered about a gigantic -cyberspace smorgasbord of "Conferences", each conference itself consisting -of a welter of "topics," each topic containing dozens, sometimes hundreds -of comments, in a tumbling, multiperson debate that could last for months -or years on end. - - -In 1991, the Well's list of conferences looked like this: - - -CONFERENCES ON THE WELL - -WELL "Screenzine" Digest (g zine) - -Best of the WELL - vintage material - (g best) - -Index listing of new topics in all conferences - (g newtops) - -Business - Education ----------------------- - -Apple Library Users Group(g alug) Agriculture (g agri) -Brainstorming (g brain) Classifieds (g cla) -Computer Journalism (g cj) Consultants (g consult) -Consumers (g cons) Design (g design) -Desktop Publishing (g desk) Disability (g disability) -Education (g ed) Energy (g energy91) -Entrepreneurs (g entre) Homeowners (g home) -Indexing (g indexing) Investments (g invest) -Kids91 (g kids) Legal (g legal) -One Person Business (g one) -Periodical/newsletter (g per) -Telecomm Law (g tcl) The Future (g fut) -Translators (g trans) Travel (g tra) -Work (g work) - -Electronic Frontier Foundation (g eff) -Computers, Freedom & Privacy (g cfp) -Computer Professionals for Social Responsibility (g cpsr) - -Social - Political - Humanities ---------------------------------- - -Aging (g gray) AIDS (g aids) -Amnesty International (g amnesty) Archives (g arc) -Berkeley (g berk) Buddhist (g wonderland) -Christian (g cross) Couples (g couples) -Current Events (g curr) Dreams (g dream) -Drugs (g dru) East Coast (g east) -Emotional Health@@@@ (g private) Erotica (g eros) -Environment (g env) Firearms (g firearms) -First Amendment (g first) Fringes of Reason (g fringes) -Gay (g gay) Gay (Private)# (g gaypriv) -Geography (g geo) German (g german) -Gulf War (g gulf) Hawaii (g aloha) -Health (g heal) History (g hist) -Holistic (g holi) Interview (g inter) -Italian (g ital) Jewish (g jew) -Liberty (g liberty) Mind (g mind) -Miscellaneous (g misc) Men on the WELL@@ (g mow) -Network Integration (g origin) Nonprofits (g non) -North Bay (g north) Northwest (g nw) -Pacific Rim (g pacrim) Parenting (g par) -Peace (g pea) Peninsula (g pen) -Poetry (g poetry) Philosophy (g phi) -Politics (g pol) Psychology (g psy) -Psychotherapy (g therapy) Recovery## (g recovery) -San Francisco (g sanfran) Scams (g scam) -Sexuality (g sex) Singles (g singles) -Southern (g south) Spanish (g spanish) -Spirituality (g spirit) Tibet (g tibet) -Transportation (g transport) True Confessions (g tru) -Unclear (g unclear) WELL Writer's Workshop@@@(g www) -Whole Earth (g we) Women on the WELL@(g wow) -Words (g words) Writers (g wri) - -@@@@Private Conference - mail wooly for entry -@@@Private conference - mail sonia for entry -@@Private conference - mail flash for entry -@ Private conference - mail reva for entry -# Private Conference - mail hudu for entry -## Private Conference - mail dhawk for entry - -Arts - Recreation - Entertainment ------------------------------------ -ArtCom Electronic Net (g acen) -Audio-Videophilia (g aud) -Bicycles (g bike) Bay Area Tonight@@(g bat) -Boating (g wet) Books (g books) -CD's (g cd) Comics (g comics) -Cooking (g cook) Flying (g flying) -Fun (g fun) Games (g games) -Gardening (g gard) Kids (g kids) -Nightowls@ (g owl) Jokes (g jokes) -MIDI (g midi) Movies (g movies) -Motorcycling (g ride) Motoring (g car) -Music (g mus) On Stage (g onstage) -Pets (g pets) Radio (g rad) -Restaurant (g rest) Science Fiction (g sf) -Sports (g spo) Star Trek (g trek) -Television (g tv) Theater (g theater) -Weird (g weird) Zines/Factsheet Five(g f5) -@Open from midnight to 6am -@@Updated daily - -Grateful Dead -------------- -Grateful Dead (g gd) Deadplan@ (g dp) -Deadlit (g deadlit) Feedback (g feedback) -GD Hour (g gdh) Tapes (g tapes) -Tickets (g tix) Tours (g tours) - -@Private conference - mail tnf for entry - -Computers ------------ -AI/Forth/Realtime (g realtime) Amiga (g amiga) -Apple (g app) Computer Books (g cbook) -Art & Graphics (g gra) Hacking (g hack) -HyperCard (g hype) IBM PC (g ibm) -LANs (g lan) Laptop (g lap) -Macintosh (g mac) Mactech (g mactech) -Microtimes (g microx) Muchomedia (g mucho) -NeXt (g next) OS/2 (g os2) -Printers (g print) Programmer's Net (g net) -Siggraph (g siggraph) Software Design (g sdc) -Software/Programming (g software) -Software Support (g ssc) -Unix (g unix) Windows (g windows) -Word Processing (g word) - -Technical - Communications ----------------------------- -Bioinfo (g bioinfo) Info (g boing) -Media (g media) NAPLPS (g naplps) -Netweaver (g netweaver) Networld (g networld) -Packet Radio (g packet) Photography (g pho) -Radio (g rad) Science (g science) -Technical Writers (g tec) Telecommunications(g tele) -Usenet (g usenet) Video (g vid) -Virtual Reality (g vr) - -The WELL Itself ---------------- -Deeper (g deeper) Entry (g ent) -General (g gentech) Help (g help) -Hosts (g hosts) Policy (g policy) -System News (g news) Test (g test) - -The list itself is dazzling, bringing to the untutored eye -a dizzying impression of a bizarre milieu of mountain-climbing -Hawaiian holistic photographers trading true-life confessions -with bisexual word-processing Tibetans. - -But this confusion is more apparent than real. Each of these conferences -was a little cyberspace world in itself, comprising dozens and perhaps -hundreds of sub-topics. Each conference was commonly frequented by -a fairly small, fairly like-minded community of perhaps a few dozen people. -It was humanly impossible to encompass the entire Well (especially since -access to the Well's mainframe computer was billed by the hour). -Most long-time users contented themselves with a few favorite -topical neighborhoods, with the occasional foray elsewhere -for a taste of exotica. But especially important news items, -and hot topical debates, could catch the attention of the entire -Well community. - -Like any community, the Well had its celebrities, and John Perry Barlow, -the silver-tongued and silver-modemed lyricist of the Grateful Dead, -ranked prominently among them. It was here on the Well that Barlow -posted his true-life tale of computer-crime encounter with the FBI. - -The story, as might be expected, created a great stir. The Well was -already primed for hacker controversy. In December 1989, Harper's magazine -had hosted a debate on the Well about the ethics of illicit computer intrusion. -While over forty various computer-mavens took part, Barlow proved a star -in the debate. So did "Acid Phreak" and "Phiber Optik," a pair of young -New York hacker-phreaks whose skills at telco switching-station intrusion -were matched only by their apparently limitless hunger for fame. -The advent of these two boldly swaggering outlaws in the precincts -of the Well created a sensation akin to that of Black Panthers -at a cocktail party for the radically chic. - -Phiber Optik in particular was to seize the day in 1990. -A devotee of the 2600 circle and stalwart of the New York -hackers' group "Masters of Deception," Phiber Optik was -a splendid exemplar of the computer intruder as committed dissident. -The eighteen-year-old Optik, a high-school dropout and part-time -computer repairman, was young, smart, and ruthlessly obsessive, -a sharp-dressing, sharp-talking digital dude who was utterly -and airily contemptuous of anyone's rules but his own. -By late 1991, Phiber Optik had appeared in Harper's, -Esquire, The New York Times, in countless public debates -and conventions, even on a television show hosted by Geraldo Rivera. - -Treated with gingerly respect by Barlow and other Well mavens, -Phiber Optik swiftly became a Well celebrity. Strangely, despite -his thorny attitude and utter single-mindedness, Phiber Optik seemed -to arouse strong protective instincts in most of the people who met him. -He was great copy for journalists, always fearlessly ready to swagger, -and, better yet, to actually DEMONSTRATE some off-the-wall digital stunt. -He was a born media darling. - -Even cops seemed to recognize that there was something peculiarly unworldly -and uncriminal about this particular troublemaker. He was so bold, -so flagrant, so young, and so obviously doomed, that even those -who strongly disapproved of his actions grew anxious for his welfare, -and began to flutter about him as if he were an endangered seal pup. - -In January 24, 1990 (nine days after the Martin Luther King Day Crash), -Phiber Optik, Acid Phreak, and a third NYC scofflaw named Scorpion were -raided by the Secret Service. Their computers went out the door, -along with the usual blizzard of papers, notebooks, compact disks, -answering machines, Sony Walkmans, etc. Both Acid Phreak and -Phiber Optik were accused of having caused the Crash. - -The mills of justice ground slowly. The case eventually fell into -the hands of the New York State Police. Phiber had lost his machinery -in the raid, but there were no charges filed against him for over a year. -His predicament was extensively publicized on the Well, where it caused -much resentment for police tactics. It's one thing to merely hear about -a hacker raided or busted; it's another to see the police attacking someone -you've come to know personally, and who has explained his motives at length. -Through the Harper's debate on the Well, it had become clear to the -Wellbeings that Phiber Optik was not in fact going to "hurt anything." -In their own salad days, many Wellbeings had tasted tear-gas in pitched -street-battles with police. They were inclined to indulgence for -acts of civil disobedience. - -Wellbeings were also startled to learn of the draconian thoroughness -of a typical hacker search-and-seizure. It took no great stretch of -imagination for them to envision themselves suffering much the same treatment. - -As early as January 1990, sentiment on the Well had already begun to sour, -and people had begun to grumble that "hackers" were getting a raw deal -from the ham-handed powers-that-be. The resultant issue of Harper's -magazine posed the question as to whether computer-intrusion was a "crime" -at all. As Barlow put it later: "I've begun to wonder if we wouldn't -also regard spelunkers as desperate criminals if AT&T owned all the caves." - -In February 1991, more than a year after the raid on his home, -Phiber Optik was finally arrested, and was charged with first-degree -Computer Tampering and Computer Trespass, New York state offenses. -He was also charged with a theft-of-service misdemeanor, involving a complex -free-call scam to a 900 number. Phiber Optik pled guilty to the misdemeanor -charge, and was sentenced to 35 hours of community service. - -This passing harassment from the unfathomable world of straight people -seemed to bother Optik himself little if at all. Deprived of his computer -by the January search-and-seizure, he simply bought himself a portable -computer so the cops could no longer monitor the phone where he lived -with his Mom, and he went right on with his depredations, sometimes on -live radio or in front of television cameras. - -The crackdown raid may have done little to dissuade Phiber Optik, -but its galling affect on the Wellbeings was profound. As 1990 rolled on, -the slings and arrows mounted: the Knight Lightning raid, -the Steve Jackson raid, the nation-spanning Operation Sundevil. -The rhetoric of law enforcement made it clear that there was, -in fact, a concerted crackdown on hackers in progress. - -The hackers of the Hackers Conference, the Wellbeings, and their ilk, -did not really mind the occasional public misapprehension of "hacking;" -if anything, this membrane of differentiation from straight society -made the "computer community" feel different, smarter, better. -They had never before been confronted, however, by a concerted -vilification campaign. - -Barlow's central role in the counter-struggle was one of the major -anomalies of 1990. Journalists investigating the controversy -often stumbled over the truth about Barlow, but they commonly -dusted themselves off and hurried on as if nothing had happened. -It was as if it were TOO MUCH TO BELIEVE that a 1960s freak -from the Grateful Dead had taken on a federal law enforcement operation -head-to-head and ACTUALLY SEEMED TO BE WINNING! - -Barlow had no easily detectable power-base for a political struggle -of this kind. He had no formal legal or technical credentials. -Barlow was, however, a computer networker of truly stellar brilliance. -He had a poet's gift of concise, colorful phrasing. He also had a -journalist's shrewdness, an off-the-wall, self-deprecating wit, -and a phenomenal wealth of simple personal charm. - -The kind of influence Barlow possessed is fairly common currency -in literary, artistic, or musical circles. A gifted critic can -wield great artistic influence simply through defining -the temper of the times, by coining the catch-phrases -and the terms of debate that become the common currency of the period. -(And as it happened, Barlow WAS a part-time art critic, -with a special fondness for the Western art of Frederic Remington.) - -Barlow was the first commentator to adopt William Gibson's -striking science-fictional term "cyberspace" as a synonym -for the present-day nexus of computer and telecommunications networks. -Barlow was insistent that cyberspace should be regarded as -a qualitatively new world, a "frontier." According to Barlow, -the world of electronic communications, now made visible through -the computer screen, could no longer be usefully regarded -as just a tangle of high-tech wiring. Instead, it had become -a PLACE, cyberspace, which demanded a new set of metaphors, -a new set of rules and behaviors. The term, as Barlow employed it, -struck a useful chord, and this concept of cyberspace was picked up -by Time, Scientific American, computer police, hackers, and even -Constitutional scholars. "Cyberspace" now seems likely to become -a permanent fixture of the language. - -Barlow was very striking in person: a tall, craggy-faced, bearded, -deep-voiced Wyomingan in a dashing Western ensemble of jeans, jacket, -cowboy boots, a knotted throat-kerchief and an ever-present Grateful Dead -cloisonne lapel pin. - -Armed with a modem, however, Barlow was truly in his element. -Formal hierarchies were not Barlow's strong suit; he rarely missed -a chance to belittle the "large organizations and their drones," -with their uptight, institutional mindset. Barlow was very much -of the free-spirit persuasion, deeply unimpressed by brass-hats -and jacks-in-office. But when it came to the digital grapevine, -Barlow was a cyberspace ad-hocrat par excellence. - -There was not a mighty army of Barlows. There was only one Barlow, -and he was a fairly anomolous individual. However, the situation only -seemed to REQUIRE a single Barlow. In fact, after 1990, many people -must have concluded that a single Barlow was far more than -they'd ever bargained for. - -Barlow's querulous mini-essay about his encounter with the FBI -struck a strong chord on the Well. A number of other free spirits -on the fringes of Apple Computing had come under suspicion, -and they liked it not one whit better than he did. - -One of these was Mitchell Kapor, the co-inventor of the spreadsheet -program "Lotus 1-2-3" and the founder of Lotus Development Corporation. -Kapor had written-off the passing indignity of being fingerprinted -down at his own local Boston FBI headquarters, but Barlow's post -made the full national scope of the FBI's dragnet clear to Kapor. -The issue now had Kapor's full attention. As the Secret Service -swung into anti-hacker operation nationwide in 1990, Kapor watched -every move with deep skepticism and growing alarm. - -As it happened, Kapor had already met Barlow, who had interviewed Kapor -for a California computer journal. Like most people who met Barlow, -Kapor had been very taken with him. Now Kapor took it upon himself -to drop in on Barlow for a heart-to-heart talk about the situation. - -Kapor was a regular on the Well. Kapor had been a devotee of the -Whole Earth Catalogsince the beginning, and treasured a complete run -of the magazine. And Kapor not only had a modem, but a private jet. -In pursuit of the scattered high-tech investments of Kapor Enterprises Inc., -his personal, multi-million dollar holding company, Kapor commonly crossed -state lines with about as much thought as one might give to faxing a letter. - -The Kapor-Barlow council of June 1990, in Pinedale, Wyoming, was the start -of the Electronic Frontier Foundation. Barlow swiftly wrote a manifesto, -"Crime and Puzzlement," which announced his, and Kapor's, intention -to form a political organization to "raise and disburse funds for education, -lobbying, and litigation in the areas relating to digital speech and the -extension of the Constitution into Cyberspace." - -Furthermore, proclaimed the manifesto, the foundation would -"fund, conduct, and support legal efforts to demonstrate -that the Secret Service has exercised prior restraint on publications, -limited free speech, conducted improper seizure of equipment and data, -used undue force, and generally conducted itself in a fashion which -is arbitrary, oppressive, and unconstitutional." - -"Crime and Puzzlement" was distributed far and wide through computer -networking channels, and also printed in the Whole Earth Review. -The sudden declaration of a coherent, politicized counter-strike -from the ranks of hackerdom electrified the community. Steve Wozniak -(perhaps a bit stung by the NuPrometheus scandal) swiftly offered -to match any funds Kapor offered the Foundation. - -John Gilmore, one of the pioneers of Sun Microsystems, immediately offered -his own extensive financial and personal support. Gilmore, an ardent -libertarian, was to prove an eloquent advocate of electronic privacy issues, -especially freedom from governmental and corporate computer-assisted -surveillance of private citizens. - -A second meeting in San Francisco rounded up further allies: -Stewart Brand of the Point Foundation, virtual-reality pioneers -Jaron Lanier and Chuck Blanchard, network entrepreneur and venture -capitalist Nat Goldhaber. At this dinner meeting, the activists settled on -a formal title: the Electronic Frontier Foundation, Incorporated. -Kapor became its president. A new EFF Conference was opened on -the Point Foundation's Well, and the Well was declared -"the home of the Electronic Frontier Foundation." - -Press coverage was immediate and intense. Like their -nineteenth-century spiritual ancestors, Alexander Graham Bell -and Thomas Watson, the high-tech computer entrepreneurs -of the 1970s and 1980s--people such as Wozniak, Jobs, Kapor, -Gates, and H. Ross Perot, who had raised themselves by their bootstraps -to dominate a glittering new industry--had always made very good copy. - -But while the Wellbeings rejoiced, the press in general seemed -nonplussed by the self-declared "civilizers of cyberspace." -EFF's insistence that the war against "hackers" involved grave -Constitutional civil liberties issues seemed somewhat farfetched, -especially since none of EFF's organizers were lawyers -or established politicians. The business press in particular -found it easier to seize on the apparent core of the story-- -that high-tech entrepreneur Mitchell Kapor had established -a "defense fund for hackers." Was EFF a genuinely important -political development--or merely a clique of wealthy eccentrics, -dabbling in matters better left to the proper authorities? -The jury was still out. - -But the stage was now set for open confrontation. -And the first and the most critical battle was the -hacker show-trial of "Knight Lightning." - -# - -It has been my practice throughout this book to refer to hackers -only by their "handles." There is little to gain by giving -the real names of these people, many of whom are juveniles, -many of whom have never been convicted of any crime, and many -of whom had unsuspecting parents who have already suffered enough. - -But the trial of Knight Lightning on July 24-27, 1990, -made this particular "hacker" a nationally known public figure. -It can do no particular harm to himself or his family if I repeat -the long-established fact that his name is Craig Neidorf (pronounced NYE-dorf). - -Neidorf's jury trial took place in the United States District Court, -Northern District of Illinois, Eastern Division, with the -Honorable Nicholas J. Bua presiding. The United States of America -was the plaintiff, the defendant Mr. Neidorf. The defendant's attorney -was Sheldon T. Zenner of the Chicago firm of Katten, Muchin and Zavis. - -The prosecution was led by the stalwarts of the Chicago Computer Fraud -and Abuse Task Force: William J. Cook, Colleen D. Coughlin, and -David A. Glockner, all Assistant United States Attorneys. -The Secret Service Case Agent was Timothy M. Foley. - -It will be recalled that Neidorf was the co-editor of an underground hacker -"magazine" called Phrack. Phrack was an entirely electronic publication, -distributed through bulletin boards and over electronic networks. -It was amateur publication given away for free. Neidorf had never made -any money for his work in Phrack. Neither had his unindicted co-editor -"Taran King" or any of the numerous Phrack contributors. - -The Chicago Computer Fraud and Abuse Task Force, however, -had decided to prosecute Neidorf as a fraudster. -To formally admit that Phrack was a "magazine" -and Neidorf a "publisher" was to open a prosecutorial -Pandora's Box of First Amendment issues. To do this -was to play into the hands of Zenner and his EFF advisers, -which now included a phalanx of prominent New York civil rights -lawyers as well as the formidable legal staff of Katten, Muchin and Zavis. -Instead, the prosecution relied heavily on the issue of access device fraud: -Section 1029 of Title 18, the section from which the Secret Service drew -its most direct jurisdiction over computer crime. - -Neidorf's alleged crimes centered around the E911 Document. -He was accused of having entered into a fraudulent scheme with the Prophet, -who, it will be recalled, was the Atlanta LoD member who had illicitly -copied the E911 Document from the BellSouth AIMSX system. - -The Prophet himself was also a co-defendant in the Neidorf case, -part-and-parcel of the alleged "fraud scheme" to "steal" BellSouth's -E911 Document (and to pass the Document across state lines, -which helped establish the Neidorf trial as a federal case). -The Prophet, in the spirit of full co-operation, had agreed -to testify against Neidorf. - -In fact, all three of the Atlanta crew stood ready to testify against Neidorf. -Their own federal prosecutors in Atlanta had charged the Atlanta Three with: -(a) conspiracy, (b) computer fraud, (c) wire fraud, (d) access device fraud, -and (e) interstate transportation of stolen property (Title 18, Sections 371, -1030, 1343, 1029, and 2314). - -Faced with this blizzard of trouble, Prophet and Leftist had ducked -any public trial and had pled guilty to reduced charges--one conspiracy -count apiece. Urvile had pled guilty to that odd bit of Section 1029 -which makes it illegal to possess "fifteen or more" illegal access devices -(in his case, computer passwords). And their sentences were scheduled -for September 14, 1990--well after the Neidorf trial. As witnesses, -they could presumably be relied upon to behave. - -Neidorf, however, was pleading innocent. Most everyone else caught up -in the crackdown had "cooperated fully" and pled guilty in hope -of reduced sentences. (Steve Jackson was a notable exception, -of course, and had strongly protested his innocence from the -very beginning. But Steve Jackson could not get a day in court-- -Steve Jackson had never been charged with any crime in the first place.) - -Neidorf had been urged to plead guilty. But Neidorf was a political science -major and was disinclined to go to jail for "fraud" when he had not made -any money, had not broken into any computer, and had been publishing -a magazine that he considered protected under the First Amendment. - -Neidorf's trial was the ONLY legal action of the entire Crackdown -that actually involved bringing the issues at hand out for a public test -in front of a jury of American citizens. - -Neidorf, too, had cooperated with investigators. He had voluntarily -handed over much of the evidence that had led to his own indictment. -He had already admitted in writing that he knew that the E911 Document -had been stolen before he had "published" it in Phrack--or, from the -prosecution's point of view, illegally transported stolen property by wire -in something purporting to be a "publication." - -But even if the "publication" of the E911 Document was not held to be a crime, -that wouldn't let Neidorf off the hook. Neidorf had still received -the E911 Document when Prophet had transferred it to him from Rich Andrews' -Jolnet node. On that occasion, it certainly hadn't been "published"-- -it was hacker booty, pure and simple, transported across state lines. - -The Chicago Task Force led a Chicago grand jury to indict Neidorf -on a set of charges that could have put him in jail for thirty years. -When some of these charges were successfully challenged before Neidorf -actually went to trial, the Chicago Task Force rearranged his -indictment so that he faced a possible jail term of over sixty years! -As a first offender, it was very unlikely that Neidorf would in fact -receive a sentence so drastic; but the Chicago Task Force clearly -intended to see Neidorf put in prison, and his conspiratorial "magazine" -put permanently out of commission. This was a federal case, and Neidorf -was charged with the fraudulent theft of property worth almost -eighty thousand dollars. - -William Cook was a strong believer in high-profile prosecutions -with symbolic overtones. He often published articles on his work -in the security trade press, arguing that "a clear message had -to be sent to the public at large and the computer community -in particular that unauthorized attacks on computers and the theft -of computerized information would not be tolerated by the courts." - -The issues were complex, the prosecution's tactics somewhat unorthodox, -but the Chicago Task Force had proved sure-footed to date. "Shadowhawk" -had been bagged on the wing in 1989 by the Task Force, and sentenced -to nine months in prison, and a $10,000 fine. The Shadowhawk case involved -charges under Section 1030, the "federal interest computer" section. - -Shadowhawk had not in fact been a devotee of "federal-interest" computers -per se. On the contrary, Shadowhawk, who owned an AT&T home computer, -seemed to cherish a special aggression toward AT&T. He had bragged on -the underground boards "Phreak Klass 2600" and "Dr. Ripco" of his skills -at raiding AT&T, and of his intention to crash AT&T's national phone system. -Shadowhawk's brags were noticed by Henry Kluepfel of Bellcore Security, -scourge of the outlaw boards, whose relations with the Chicago Task Force -were long and intimate. - -The Task Force successfully established that Section 1030 applied to -the teenage Shadowhawk, despite the objections of his defense attorney. -Shadowhawk had entered a computer "owned" by U.S. Missile Command -and merely "managed" by AT&T. He had also entered an AT&T computer -located at Robbins Air Force Base in Georgia. Attacking AT&T was -of "federal interest" whether Shadowhawk had intended it or not. - -The Task Force also convinced the court that a piece of AT&T -software that Shadowhawk had illicitly copied from Bell Labs, -the "Artificial Intelligence C5 Expert System," was worth a cool -one million dollars. Shadowhawk's attorney had argued that -Shadowhawk had not sold the program and had made no profit from -the illicit copying. And in point of fact, the C5 Expert System -was experimental software, and had no established market value -because it had never been on the market in the first place. -AT&T's own assessment of a "one million dollar" figure for its -own intangible property was accepted without challenge -by the court, however. And the court concurred with -the government prosecutors that Shadowhawk showed clear -"intent to defraud" whether he'd gotten any money or not. -Shadowhawk went to jail. - -The Task Force's other best-known triumph had been the conviction -and jailing of "Kyrie." Kyrie, a true denizen of the digital -criminal underground, was a 36-year-old Canadian woman, -convicted and jailed for telecommunications fraud in Canada. -After her release from prison, she had fled the wrath of Canada Bell -and the Royal Canadian Mounted Police, and eventually settled, -very unwisely, in Chicago. - -"Kyrie," who also called herself "Long Distance Information," -specialized in voice-mail abuse. She assembled large numbers -of hot long-distance codes, then read them aloud into a series -of corporate voice-mail systems. Kyrie and her friends were -electronic squatters in corporate voice-mail systems, -using them much as if they were pirate bulletin boards, -then moving on when their vocal chatter clogged the system -and the owners necessarily wised up. Kyrie's camp followers -were a loose tribe of some hundred and fifty phone-phreaks, -who followed her trail of piracy from machine to machine, -ardently begging for her services and expertise. - -Kyrie's disciples passed her stolen credit-card numbers, -in exchange for her stolen "long distance information." -Some of Kyrie's clients paid her off in cash, by scamming -credit-card cash advances from Western Union. - -Kyrie travelled incessantly, mostly through airline tickets -and hotel rooms that she scammed through stolen credit cards. -Tiring of this, she found refuge with a fellow female phone -phreak in Chicago. Kyrie's hostess, like a surprising number -of phone phreaks, was blind. She was also physically disabled. -Kyrie allegedly made the best of her new situation by applying for, -and receiving, state welfare funds under a false identity as -a qualified caretaker for the handicapped. - -Sadly, Kyrie's two children by a former marriage had also vanished -underground with her; these pre-teen digital refugees had no legal -American identity, and had never spent a day in school. - -Kyrie was addicted to technical mastery and enthralled by her own -cleverness and the ardent worship of her teenage followers. -This foolishly led her to phone up Gail Thackeray in Arizona, -to boast, brag, strut, and offer to play informant. -Thackeray, however, had already learned far more -than enough about Kyrie, whom she roundly despised -as an adult criminal corrupting minors, a "female Fagin." -Thackeray passed her tapes of Kyrie's boasts to the Secret Service. - -Kyrie was raided and arrested in Chicago in May 1989. -She confessed at great length and pled guilty. - -In August 1990, Cook and his Task Force colleague Colleen Coughlin -sent Kyrie to jail for 27 months, for computer and telecommunications fraud. -This was a markedly severe sentence by the usual wrist-slapping standards -of "hacker" busts. Seven of Kyrie's foremost teenage disciples were also -indicted and convicted. The Kyrie "high-tech street gang," as Cook -described it, had been crushed. Cook and his colleagues had been -the first ever to put someone in prison for voice-mail abuse. -Their pioneering efforts had won them attention and kudos. - -In his article on Kyrie, Cook drove the message home to the readers -of Security Management magazine, a trade journal for corporate -security professionals. The case, Cook said, and Kyrie's stiff sentence, -"reflect a new reality for hackers and computer crime victims in the -'90s. . . . Individuals and corporations who report computer -and telecommunications crimes can now expect that their cooperation -with federal law enforcement will result in meaningful punishment. -Companies and the public at large must report computer-enhanced -crimes if they want prosecutors and the course to protect their rights -to the tangible and intangible property developed and stored on computers." - -Cook had made it his business to construct this "new reality for hackers." -He'd also made it his business to police corporate property rights -to the intangible. - -Had the Electronic Frontier Foundation been a "hacker defense fund" -as that term was generally understood, they presumably would have stood up -for Kyrie. Her 1990 sentence did indeed send a "message" that federal heat -was coming down on "hackers." But Kyrie found no defenders at EFF, -or anywhere else, for that matter. EFF was not a bail-out fund -for electronic crooks. - -The Neidorf case paralleled the Shadowhawk case in certain ways. -The victim once again was allowed to set the value of the "stolen" property. -Once again Kluepfel was both investigator and technical advisor. -Once again no money had changed hands, but the "intent to defraud" was central. - -The prosecution's case showed signs of weakness early on. The Task Force -had originally hoped to prove Neidorf the center of a nationwide -Legion of Doom criminal conspiracy. The Phrack editors threw physical -get-togethers every summer, which attracted hackers from across the country; -generally two dozen or so of the magazine's favorite contributors and readers. -(Such conventions were common in the hacker community; 2600 Magazine, -for instance, held public meetings of hackers in New York, every month.) -LoD heavy-dudes were always a strong presence at these Phrack-sponsored -"Summercons." - -In July 1988, an Arizona hacker named "Dictator" attended Summercon -in Neidorf's home town of St. Louis. Dictator was one of Gail Thackeray's -underground informants; Dictator's underground board in Phoenix was -a sting operation for the Secret Service. Dictator brought an undercover -crew of Secret Service agents to Summercon. The agents bored spyholes -through the wall of Dictator's hotel room in St Louis, and videotaped -the frolicking hackers through a one-way mirror. As it happened, -however, nothing illegal had occurred on videotape, other than the -guzzling of beer by a couple of minors. Summercons were social events, -not sinister cabals. The tapes showed fifteen hours of raucous laughter, -pizza-gobbling, in-jokes and back-slapping. - -Neidorf's lawyer, Sheldon Zenner, saw the Secret Service tapes -before the trial. Zenner was shocked by the complete harmlessness -of this meeting, which Cook had earlier characterized as a sinister -interstate conspiracy to commit fraud. Zenner wanted to show the -Summercon tapes to the jury. It took protracted maneuverings -by the Task Force to keep the tapes from the jury as "irrelevant." - -The E911 Document was also proving a weak reed. It had originally -been valued at $79,449. Unlike Shadowhawk's arcane Artificial Intelligence -booty, the E911 Document was not software--it was written in English. -Computer-knowledgeable people found this value--for a twelve-page -bureaucratic document--frankly incredible. In his "Crime and Puzzlement" -manifesto for EFF, Barlow commented: "We will probably never know how -this figure was reached or by whom, though I like to imagine an appraisal -team consisting of Franz Kafka, Joseph Heller, and Thomas Pynchon." - -As it happened, Barlow was unduly pessimistic. The EFF did, in fact, -eventually discover exactly how this figure was reached, and by whom-- -but only in 1991, long after the Neidorf trial was over. - -Kim Megahee, a Southern Bell security manager, -had arrived at the document's value by simply adding up -the "costs associated with the production" of the E911 Document. -Those "costs" were as follows: - -1. A technical writer had been hired to research and write the E911 Document. - 200 hours of work, at $35 an hour, cost : $7,000. A Project Manager had - overseen the technical writer. 200 hours, at $31 an hour, made: $6,200. - -2. A week of typing had cost $721 dollars. A week of formatting had - cost $721. A week of graphics formatting had cost $742. - -3. Two days of editing cost $367. - -4. A box of order labels cost five dollars. - -5. Preparing a purchase order for the Document, including typing - and the obtaining of an authorizing signature from within the - BellSouth bureaucracy, cost $129. - -6. Printing cost $313. Mailing the Document to fifty people - took fifty hours by a clerk, and cost $858. - -7. Placing the Document in an index took two clerks an hour each, - totalling $43. - -Bureaucratic overhead alone, therefore, was alleged to have cost -a whopping $17,099. According to Mr. Megahee, the typing -of a twelve-page document had taken a full week. Writing it -had taken five weeks, including an overseer who apparently -did nothing else but watch the author for five weeks. -Editing twelve pages had taken two days. Printing and mailing -an electronic document (which was already available on the -Southern Bell Data Network to any telco employee who needed it), -had cost over a thousand dollars. - -But this was just the beginning. There were also the HARDWARE EXPENSES. -Eight hundred fifty dollars for a VT220 computer monitor. -THIRTY-ONE THOUSAND DOLLARS for a sophisticated VAXstation II computer. -Six thousand dollars for a computer printer. TWENTY-TWO THOUSAND DOLLARS -for a copy of "Interleaf" software. Two thousand five hundred dollars -for VMS software. All this to create the twelve-page Document. - -Plus ten percent of the cost of the software and the hardware, for maintenance. -(Actually, the ten percent maintenance costs, though mentioned, had been left -off the final $79,449 total, apparently through a merciful oversight). - -Mr. Megahee's letter had been mailed directly to William Cook himself, -at the office of the Chicago federal attorneys. The United States Government -accepted these telco figures without question. - -As incredulity mounted, the value of the E911 Document was officially -revised downward. This time, Robert Kibler of BellSouth Security -estimated the value of the twelve pages as a mere $24,639.05--based, -purportedly, on "R&D costs." But this specific estimate, -right down to the nickel, did not move the skeptics at all; -in fact it provoked open scorn and a torrent of sarcasm. - -The financial issues concerning theft of proprietary information -have always been peculiar. It could be argued that BellSouth -had not "lost" its E911 Document at all in the first place, -and therefore had not suffered any monetary damage from this "theft." -And Sheldon Zenner did in fact argue this at Neidorf's trial-- -that Prophet's raid had not been "theft," but was better understood -as illicit copying. - -The money, however, was not central to anyone's true purposes in this trial. -It was not Cook's strategy to convince the jury that the E911 Document -was a major act of theft and should be punished for that reason alone. -His strategy was to argue that the E911 Document was DANGEROUS. -It was his intention to establish that the E911 Document was "a road-map" -to the Enhanced 911 System. Neidorf had deliberately and recklessly -distributed a dangerous weapon. Neidorf and the Prophet did not care -(or perhaps even gloated at the sinister idea) that the E911 Document -could be used by hackers to disrupt 911 service, "a life line for every -person certainly in the Southern Bell region of the United States, -and indeed, in many communities throughout the United States," -in Cook's own words. Neidorf had put people's lives in danger. - -In pre-trial maneuverings, Cook had established that the E911 Document -was too hot to appear in the public proceedings of the Neidorf trial. -The JURY ITSELF would not be allowed to ever see this Document, -lest it slip into the official court records, and thus into the hands -of the general public, and, thus, somehow, to malicious hackers -who might lethally abuse it. - -Hiding the E911 Document from the jury may have been a -clever legal maneuver, but it had a severe flaw. There were, -in point of fact, hundreds, perhaps thousands, of people, -already in possession of the E911 Document, just as Phrack -had published it. Its true nature was already obvious -to a wide section of the interested public (all of whom, -by the way, were, at least theoretically, party to -a gigantic wire-fraud conspiracy). Most everyone -in the electronic community who had a modem and any -interest in the Neidorf case already had a copy of the Document. -It had already been available in Phrack for over a year. - -People, even quite normal people without any particular -prurient interest in forbidden knowledge, did not shut their eyes -in terror at the thought of beholding a "dangerous" document -from a telephone company. On the contrary, they tended to trust -their own judgement and simply read the Document for themselves. -And they were not impressed. - -One such person was John Nagle. Nagle was a forty-one-year-old -professional programmer with a masters' degree in computer science -from Stanford. He had worked for Ford Aerospace, where he had invented -a computer-networking technique known as the "Nagle Algorithm," -and for the prominent Californian computer-graphics firm "Autodesk," -where he was a major stockholder. - -Nagle was also a prominent figure on the Well, much respected -for his technical knowledgeability. - -Nagle had followed the civil-liberties debate closely, -for he was an ardent telecommunicator. He was no particular friend -of computer intruders, but he believed electronic publishing -had a great deal to offer society at large, and attempts -to restrain its growth, or to censor free electronic expression, -strongly roused his ire. - -The Neidorf case, and the E911 Document, were both being discussed -in detail on the Internet, in an electronic publication called Telecom Digest. -Nagle, a longtime Internet maven, was a regular reader of Telecom Digest. -Nagle had never seen a copy of Phrack, but the implications of the case -disturbed him. - -While in a Stanford bookstore hunting books on robotics, -Nagle happened across a book called The Intelligent Network. -Thumbing through it at random, Nagle came across an entire chapter -meticulously detailing the workings of E911 police emergency systems. -This extensive text was being sold openly, and yet in Illinois -a young man was in danger of going to prison for publishing -a thin six-page document about 911 service. - -Nagle made an ironic comment to this effect in Telecom Digest. -From there, Nagle was put in touch with Mitch Kapor, -and then with Neidorf's lawyers. - -Sheldon Zenner was delighted to find a computer telecommunications expert -willing to speak up for Neidorf, one who was not a wacky teenage "hacker." -Nagle was fluent, mature, and respectable; he'd once had a federal -security clearance. - -Nagle was asked to fly to Illinois to join the defense team. - -Having joined the defense as an expert witness, Nagle read the entire -E911 Document for himself. He made his own judgement about its potential -for menace. - -The time has now come for you yourself, the reader, to have a look -at the E911 Document. This six-page piece of work was the pretext -for a federal prosecution that could have sent an electronic publisher -to prison for thirty, or even sixty, years. It was the pretext -for the search and seizure of Steve Jackson Games, a legitimate publisher -of printed books. It was also the formal pretext for the search -and seizure of the Mentor's bulletin board, "Phoenix Project," -and for the raid on the home of Erik Bloodaxe. It also had much -to do with the seizure of Richard Andrews' Jolnet node -and the shutdown of Charles Boykin's AT&T node. -The E911 Document was the single most important piece -of evidence in the Hacker Crackdown. There can be no real -and legitimate substitute for the Document itself. - - -==Phrack Inc.== - -Volume Two, Issue 24, File 5 of 13 - -Control Office Administration -Of Enhanced 911 Services For -Special Services and Account Centers - -by the Eavesdropper - -March, 1988 - - -Description of Service -~~~~~~~~~~~~~~~~~~~~~ -The control office for Emergency 911 service is assigned in -accordance with the existing standard guidelines to one of -the following centers: - -o Special Services Center (SSC) -o Major Accounts Center (MAC) -o Serving Test Center (STC) -o Toll Control Center (TCC) - -The SSC/MAC designation is used in this document interchangeably -for any of these four centers. The Special Services Centers (SSCs) -or Major Account Centers (MACs) have been designated as the trouble -reporting contact for all E911 customer (PSAP) reported troubles. -Subscribers who have trouble on an E911 call will continue -to contact local repair service (CRSAB) who will refer the -trouble to the SSC/MAC, when appropriate. - -Due to the critical nature of E911 service, the control -and timely repair of troubles is demanded. As the primary -E911 customer contact, the SSC/MAC is in the unique position -to monitor the status of the trouble and insure its resolution. - -System Overview -~~~~~~~~~~~~~~ -The number 911 is intended as a nationwide universal -telephone number which provides the public with direct -access to a Public Safety Answering Point (PSAP). A PSAP -is also referred to as an Emergency Service Bureau (ESB). -A PSAP is an agency or facility which is authorized by a -municipality to receive and respond to police, fire and/or -ambulance services. One or more attendants are located -at the PSAP facilities to receive and handle calls of an -emergency nature in accordance with the local municipal -requirements. - -An important advantage of E911 emergency service is -improved (reduced) response times for emergency -services. Also close coordination among agencies -providing various emergency services is a valuable -capability provided by E911 service. - -1A ESS is used as the tandem office for the E911 network to -route all 911 calls to the correct (primary) PSAP designated -to serve the calling station. The E911 feature was -developed primarily to provide routing to the correct PSAP -for all 911 calls. Selective routing allows a 911 call -originated from a particular station located in a particular -district, zone, or town, to be routed to the primary PSAP -designated to serve that customer station regardless of -wire center boundaries. Thus, selective routing eliminates -the problem of wire center boundaries not coinciding with -district or other political boundaries. - -The services available with the E911 feature include: - -Forced Disconnect Default Routing -Alternative Routing Night Service -Selective Routing Automatic Number -Identification (ANI) -Selective Transfer Automatic Location -Identification (ALI) - - -Preservice/Installation Guidelines -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -When a contract for an E911 system has been signed, it is -the responsibility of Network Marketing to establish an -implementation/cutover committee which should include -a representative from the SSC/MAC. Duties of the E911 -Implementation Team include coordination of all phases -of the E911 system deployment and the formation of an -on-going E911 maintenance subcommittee. - -Marketing is responsible for providing the following -customer specific information to the SSC/MAC prior to -the start of call through testing: - -o All PSAP's (name, address, local contact) -o All PSAP circuit ID's -o 1004 911 service request including PSAP details on each PSAP - (1004 Section K, L, M) -o Network configuration -o Any vendor information (name, telephone number, equipment) - -The SSC/MAC needs to know if the equipment and sets -at the PSAP are maintained by the BOCs, an independent -company, or an outside vendor, or any combination. -This information is then entered on the PSAP profile sheets -and reviewed quarterly for changes, additions and deletions. - -Marketing will secure the Major Account Number (MAN) -and provide this number to Corporate Communications -so that the initial issue of the service orders carry -the MAN and can be tracked by the SSC/MAC via CORDNET. -PSAP circuits are official services by definition. - -All service orders required for the installation of the E911 -system should include the MAN assigned to the city/county -which has purchased the system. - -In accordance with the basic SSC/MAC strategy for provisioning, -the SSC/MAC will be Overall Control Office (OCO) for all Node -to PSAP circuits (official services) and any other services -for this customer. Training must be scheduled for all SSC/MAC -involved personnel during the pre-service stage of the project. - -The E911 Implementation Team will form the on-going -maintenance subcommittee prior to the initial -implementation of the E911 system. This sub-committee -will establish post implementation quality assurance -procedures to ensure that the E911 system continues to -provide quality service to the customer. -Customer/Company training, trouble reporting interfaces -for the customer, telephone company and any involved -independent telephone companies needs to be addressed -and implemented prior to E911 cutover. These functions -can be best addressed by the formation of a sub- -committee of the E911 Implementation Team to set up -guidelines for and to secure service commitments of -interfacing organizations. A SSC/MAC supervisor should -chair this subcommittee and include the following -organizations: - -1) Switching Control Center - - E911 translations - - Trunking - - End office and Tandem office hardware/software -2) Recent Change Memory Administration Center - - Daily RC update activity for TN/ESN translations - - Processes validity errors and rejects -3) Line and Number Administration - - Verification of TN/ESN translations -4) Special Service Center/Major Account Center - - Single point of contact for all PSAP and Node to host troubles - - Logs, tracks & statusing of all trouble reports - - Trouble referral, follow up, and escalation - - Customer notification of status and restoration - - Analyzation of "chronic" troubles - - Testing, installation and maintenance of E911 circuits -5) Installation and Maintenance (SSIM/I&M) - - Repair and maintenance of PSAP equipment and Telco owned sets -6) Minicomputer Maintenance Operations Center - - E911 circuit maintenance (where applicable) -7) Area Maintenance Engineer - - Technical assistance on voice (CO-PSAP) network related E911 troubles - - -Maintenance Guidelines -~~~~~~~~~~~~~~~~~~~~~ -The CCNC will test the Node circuit from the 202T at the -Host site to the 202T at the Node site. Since Host to Node -(CCNC to MMOC) circuits are official company services, -the CCNC will refer all Node circuit troubles to the -SSC/MAC. The SSC/MAC is responsible for the testing -and follow up to restoration of these circuit troubles. - -Although Node to PSAP circuit are official services, the -MMOC will refer PSAP circuit troubles to the appropriate -SSC/MAC. The SSC/MAC is responsible for testing and -follow up to restoration of PSAP circuit troubles. - -The SSC/MAC will also receive reports from -CRSAB/IMC(s) on subscriber 911 troubles when they are -not line troubles. The SSC/MAC is responsible for testing -and restoration of these troubles. - -Maintenance responsibilities are as follows: - -SCC@ Voice Network (ANI to PSAP) -@SCC responsible for tandem switch - -SSIM/I&M PSAP Equipment (Modems, CIU's, sets) -Vendor PSAP Equipment (when CPE) -SSC/MAC PSAP to Node circuits, and tandem to - PSAP voice circuits (EMNT) -MMOC Node site (Modems, cables, etc) - -Note: All above work groups are required to resolve troubles -by interfacing with appropriate work groups for resolution. - -The Switching Control Center (SCC) is responsible for -E911/1AESS translations in tandem central offices. -These translations route E911 calls, selective transfer, -default routing, speed calling, etc., for each PSAP. -The SCC is also responsible for troubleshooting on -the voice network (call originating to end office tandem equipment). - -For example, ANI failures in the originating offices would -be a responsibility of the SCC. - -Recent Change Memory Administration Center (RCMAC) performs -the daily tandem translation updates (recent change) -for routing of individual telephone numbers. - -Recent changes are generated from service order activity -(new service, address changes, etc.) and compiled into -a daily file by the E911 Center (ALI/DMS E911 Computer). - -SSIM/I&M is responsible for the installation and repair of -PSAP equipment. PSAP equipment includes ANI Controller, -ALI Controller, data sets, cables, sets, and other peripheral -equipment that is not vendor owned. SSIM/I&M is responsible -for establishing maintenance test kits, complete with spare parts -for PSAP maintenance. This includes test gear, data sets, -and ANI/ALI Controller parts. - -Special Services Center (SSC) or Major Account Center -(MAC) serves as the trouble reporting contact for all -(PSAP) troubles reported by customer. The SSC/MAC -refers troubles to proper organizations for handling and -tracks status of troubles, escalating when necessary. -The SSC/MAC will close out troubles with customer. -The SSC/MAC will analyze all troubles and tracks "chronic" -PSAP troubles. - -Corporate Communications Network Center (CCNC) will -test and refer troubles on all node to host circuits. -All E911 circuits are classified as official company property. - -The Minicomputer Maintenance Operations Center -(MMOC) maintains the E911 (ALI/DMS) computer -hardware at the Host site. This MMOC is also responsible -for monitoring the system and reporting certain PSAP -and system problems to the local MMOC's, SCC's or -SSC/MAC's. The MMOC personnel also operate software -programs that maintain the TN data base under the -direction of the E911 Center. The maintenance of the -NODE computer (the interface between the PSAP and the -ALI/DMS computer) is a function of the MMOC at the -NODE site. The MMOC's at the NODE sites may also be -involved in the testing of NODE to Host circuits. -The MMOC will also assist on Host to PSAP and data network -related troubles not resolved through standard trouble -clearing procedures. - -Installation And Maintenance Center (IMC) is responsible -for referral of E911 subscriber troubles that are not subscriber -line problems. - -E911 Center - Performs the role of System Administration -and is responsible for overall operation of the E911 -computer software. The E911 Center does A-Z trouble -analysis and provides statistical information on the -performance of the system. - -This analysis includes processing PSAP inquiries (trouble -reports) and referral of network troubles. The E911 Center -also performs daily processing of tandem recent change -and provides information to the RCMAC for tandem input. -The E911 Center is responsible for daily processing -of the ALI/DMS computer data base and provides error files, -etc. to the Customer Services department for investigation and correction. -The E911 Center participates in all system implementations and on-going -maintenance effort and assists in the development of procedures, -training and education of information to all groups. - -Any group receiving a 911 trouble from the SSC/MAC should -close out the trouble with the SSC/MAC or provide a status -if the trouble has been referred to another group. -This will allow the SSC/MAC to provide a status back -to the customer or escalate as appropriate. - -Any group receiving a trouble from the Host site (MMOC -or CCNC) should close the trouble back to that group. - -The MMOC should notify the appropriate SSC/MAC -when the Host, Node, or all Node circuits are down so that -the SSC/MAC can reply to customer reports that may be -called in by the PSAPs. This will eliminate duplicate -reporting of troubles. On complete outages the MMOC -will follow escalation procedures for a Node after two (2) -hours and for a PSAP after four (4) hours. Additionally the -MMOC will notify the appropriate SSC/MAC when the -Host, Node, or all Node circuits are down. - -The PSAP will call the SSC/MAC to report E911 troubles. -The person reporting the E911 trouble may not have a -circuit I.D. and will therefore report the PSAP name and -address. Many PSAP troubles are not circuit specific. In -those instances where the caller cannot provide a circuit -I.D., the SSC/MAC will be required to determine the -circuit I.D. using the PSAP profile. Under no circumstances -will the SSC/MAC Center refuse to take the trouble. -The E911 trouble should be handled as quickly as possible, -with the SSC/MAC providing as much assistance as -possible while taking the trouble report from the caller. - -The SSC/MAC will screen/test the trouble to determine the -appropriate handoff organization based on the following criteria: - -PSAP equipment problem: SSIM/I&M -Circuit problem: SSC/MAC -Voice network problem: SCC (report trunk group number) -Problem affecting multiple PSAPs (No ALI report from -all PSAPs): Contact the MMOC to check for NODE or -Host computer problems before further testing. - -The SSC/MAC will track the status of reported troubles -and escalate as appropriate. The SSC/MAC will close out -customer/company reports with the initiating contact. -Groups with specific maintenance responsibilities, -defined above, will investigate "chronic" troubles upon -request from the SSC/MAC and the ongoing maintenance subcommittee. - -All "out of service" E911 troubles are priority one type reports. -One link down to a PSAP is considered a priority one trouble -and should be handled as if the PSAP was isolated. - -The PSAP will report troubles with the ANI controller, ALI -controller or set equipment to the SSC/MAC. - -NO ANI: Where the PSAP reports NO ANI (digital -display screen is blank) ask if this condition exists on all -screens and on all calls. It is important to differentiate -between blank screens and screens displaying 911-00XX, -or all zeroes. - -When the PSAP reports all screens on all calls, ask if there -is any voice contact with callers. If there is no voice -contact the trouble should be referred to the SCC -immediately since 911 calls are not getting through which -may require alternate routing of calls to another PSAP. - -When the PSAP reports this condition on all screens -but not all calls and has voice contact with callers, -the report should be referred to SSIM/I&M for dispatch. -The SSC/MAC should verify with the SCC that ANI -is pulsing before dispatching SSIM. - -When the PSAP reports this condition on one screen for -all calls (others work fine) the trouble should be referred -to SSIM/I&M for dispatch, because the trouble is isolated to -one piece of equipment at the customer premise. - -An ANI failure (i.e. all zeroes) indicates that the ANI has -not been received by the PSAP from the tandem office or -was lost by the PSAP ANI controller. The PSAP may -receive "02" alarms which can be caused by the ANI -controller logging more than three all zero failures on the -same trunk. The PSAP has been instructed to report this -condition to the SSC/MAC since it could indicate an -equipment trouble at the PSAP which might be affecting -all subscribers calling into the PSAP. When all zeroes are -being received on all calls or "02" alarms continue, a tester -should analyze the condition to determine the appropriate -action to be taken. The tester must perform cooperative -testing with the SCC when there appears to be a problem -on the Tandem-PSAP trunks before requesting dispatch. - -When an occasional all zero condition is reported, -the SSC/MAC should dispatch SSIM/I&M to routine -equipment on a "chronic" troublesweep. - -The PSAPs are instructed to report incidental ANI failures -to the BOC on a PSAP inquiry trouble ticket (paper) that -is sent to the Customer Services E911 group and forwarded -to E911 center when required. This usually involves only a -particular telephone number and is not a condition that -would require a report to the SSC/MAC. Multiple ANI -failures which our from the same end office (XX denotes -end office), indicate a hard trouble condition may exist -in the end office or end office tandem trunks. The PSAP will -report this type of condition to the SSC/MAC and the -SSC/MAC should refer the report to the SCC responsible -for the tandem office. NOTE: XX is the ESCO (Emergency -Service Number) associated with the incoming 911 trunks -into the tandem. It is important that the C/MAC tell the -SCC what is displayed at the PSAP (i.e. 911-0011) which -indicates to the SCC which end office is in trouble. - -Note: It is essential that the PSAP fill out inquiry form -on every ANI failure. - -The PSAP will report a trouble any time an address is not -received on an address display (screen blank) E911 call. -(If a record is not in the 911 data base or an ANI failure -is encountered, the screen will provide a display noticing -such condition). The SSC/MAC should verify with the PSAP -whether the NO ALI condition is on one screen or all screens. - -When the condition is on one screen (other screens -receive ALI information) the SSC/MAC will request -SSIM/I&M to dispatch. - -If no screens are receiving ALI information, there is usually -a circuit trouble between the PSAP and the Host computer. -The SSC/MAC should test the trouble and refer for restoral. - -Note: If the SSC/MAC receives calls from multiple -PSAP's, all of which are receiving NO ALI, there is a -problem with the Node or Node to Host circuits or the -Host computer itself. Before referring the trouble the -SSC/MAC should call the MMOC to inquire if the Node -or Host is in trouble. - -Alarm conditions on the ANI controller digital display at -the PSAP are to be reported by the PSAP's. These alarms -can indicate various trouble conditions so the SSC/MAC -should ask the PSAP if any portion of the E911 system -is not functioning properly. - -The SSC/MAC should verify with the PSAP attendant that -the equipment's primary function is answering E911 calls. -If it is, the SSC/MAC should request a dispatch SSIM/I&M. -If the equipment is not primarily used for E911, -then the SSC/MAC should advise PSAP to contact their CPE vendor. - -Note: These troubles can be quite confusing when the -PSAP has vendor equipment mixed in with equipment -that the BOC maintains. The Marketing representative -should provide the SSC/MAC information concerning any -unusual or exception items where the PSAP should -contact their vendor. This information should be included -in the PSAP profile sheets. - -ANI or ALI controller down: When the host computer sees -the PSAP equipment down and it does not come back up, -the MMOC will report the trouble to the SSC/MAC; -the equipment is down at the PSAP, a dispatch will be required. - -PSAP link (circuit) down: The MMOC will provide the -SSC/MAC with the circuit ID that the Host computer -indicates in trouble. Although each PSAP has two circuits, -when either circuit is down the condition must be treated -as an emergency since failure of the second circuit will -cause the PSAP to be isolated. - -Any problems that the MMOC identifies from the Node -location to the Host computer will be handled directly -with the appropriate MMOC(s)/CCNC. - -Note: The customer will call only when a problem is -apparent to the PSAP. When only one circuit is down to -the PSAP, the customer may not be aware there is a -trouble, even though there is one link down, -notification should appear on the PSAP screen. -Troubles called into the SSC/MAC from the MMOC -or other company employee should not be closed out -by calling the PSAP since it may result in the -customer responding that they do not have a trouble. -These reports can only be closed out by receiving -information that the trouble was fixed and by checking -with the company employee that reported the trouble. -The MMOC personnel will be able to verify that the -trouble has cleared by reviewing a printout from the host. - -When the CRSAB receives a subscriber complaint -(i.e., cannot dial 911) the RSA should obtain as much -information as possible while the customer is on the line. - -For example, what happened when the subscriber dialed 911? -The report is automatically directed to the IMC for subscriber line testing. -When no line trouble is found, the IMC will refer the trouble condition -to the SSC/MAC. The SSC/MAC will contact Customer Services E911 Group -and verify that the subscriber should be able to call 911 and obtain the ESN. -The SSC/MAC will verify the ESN via 2SCCS. When both verifications match, -the SSC/MAC will refer the report to the SCC responsible for the 911 tandem -office for investigation and resolution. The MAC is responsible for tracking -the trouble and informing the IMC when it is resolved. - - -For more information, please refer to E911 Glossary of Terms. -End of Phrack File -_____________________________________ - - -The reader is forgiven if he or she was entirely unable to read -this document. John Perry Barlow had a great deal of fun at its expense, -in "Crime and Puzzlement:" "Bureaucrat-ese of surpassing opacity. . . . -To read the whole thing straight through without entering coma requires -either a machine or a human who has too much practice thinking like one. -Anyone who can understand it fully and fluidly had altered his consciousness -beyond the ability to ever again read Blake, Whitman, or Tolstoy. . . . -the document contains little of interest to anyone who is not a student -of advanced organizational sclerosis." - -With the Document itself to hand, however, exactly as it was published -(in its six-page edited form) in Phrack, the reader may be able to verify -a few statements of fact about its nature. First, there is no software, -no computer code, in the Document. It is not computer-programming language -like FORTRAN or C++, it is English; all the sentences have nouns and verbs -and punctuation. It does not explain how to break into the E911 system. -It does not suggest ways to destroy or damage the E911 system. - -There are no access codes in the Document. There are no computer passwords. -It does not explain how to steal long distance service. It does not explain -how to break in to telco switching stations. There is nothing in it about -using a personal computer or a modem for any purpose at all, good or bad. - -Close study will reveal that this document is not about machinery. -The E911 Document is about ADMINISTRATION. It describes how one creates -and administers certain units of telco bureaucracy: -Special Service Centers and Major Account Centers (SSC/MAC). -It describes how these centers should distribute responsibility -for the E911 service, to other units of telco bureaucracy, -in a chain of command, a formal hierarchy. It describes -who answers customer complaints, who screens calls, -who reports equipment failures, who answers those reports, -who handles maintenance, who chairs subcommittees, -who gives orders, who follows orders, WHO tells WHOM what to do. -The Document is not a "roadmap" to computers. -The Document is a roadmap to PEOPLE. - -As an aid to breaking into computer systems, the Document is USELESS. -As an aid to harassing and deceiving telco people, however, the Document -might prove handy (especially with its Glossary, which I have not included). -An intense and protracted study of this Document and its Glossary, -combined with many other such documents, might teach one to speak like -a telco employee. And telco people live by SPEECH--they live by phone -communication. If you can mimic their language over the phone, -you can "social-engineer" them. If you can con telco people, you can -wreak havoc among them. You can force them to no longer trust one another; -you can break the telephonic ties that bind their community; you can make -them paranoid. And people will fight harder to defend their community -than they will fight to defend their individual selves. - -This was the genuine, gut-level threat posed by Phrack magazine. -The real struggle was over the control of telco language, -the control of telco knowledge. It was a struggle to defend the social -"membrane of differentiation" that forms the walls of the telco -community's ivory tower --the special jargon that allows telco -professionals to recognize one another, and to exclude charlatans, -thieves, and upstarts. And the prosecution brought out this fact. -They repeatedly made reference to the threat posed to telco professionals -by hackers using "social engineering." - -However, Craig Neidorf was not on trial for learning to speak like -a professional telecommunications expert. Craig Neidorf was on trial -for access device fraud and transportation of stolen property. -He was on trial for stealing a document that was purportedly -highly sensitive and purportedly worth tens of thousands of dollars. - -# - -John Nagle read the E911 Document. He drew his own conclusions. -And he presented Zenner and his defense team with an overflowing box -of similar material, drawn mostly from Stanford University's -engineering libraries. During the trial, the defense team--Zenner, -half-a-dozen other attorneys, Nagle, Neidorf, and computer-security -expert Dorothy Denning, all pored over the E911 Document line-by-line. - -On the afternoon of July 25, 1990, Zenner began to cross-examine -a woman named Billie Williams, a service manager for Southern Bell -in Atlanta. Ms. Williams had been responsible for the E911 Document. -(She was not its author--its original "author" was a Southern Bell -staff manager named Richard Helms. However, Mr. Helms should not bear -the entire blame; many telco staff people and maintenance personnel -had amended the Document. It had not been so much "written" by a -single author, as built by committee out of concrete-blocks of jargon.) - -Ms. Williams had been called as a witness for the prosecution, -and had gamely tried to explain the basic technical structure -of the E911 system, aided by charts. - -Now it was Zenner's turn. He first established that the -"proprietary stamp" that BellSouth had used on the E911 Document -was stamped on EVERY SINGLE DOCUMENT that BellSouth wrote-- -THOUSANDS of documents. "We do not publish anything other -than for our own company," Ms. Williams explained. -"Any company document of this nature is considered proprietary." -Nobody was in charge of singling out special high-security publications -for special high-security protection. They were ALL special, -no matter how trivial, no matter what their subject matter-- -the stamp was put on as soon as any document was written, -and the stamp was never removed. - -Zenner now asked whether the charts she had been using to explain -the mechanics of E911 system were "proprietary," too. -Were they PUBLIC INFORMATION, these charts, all about PSAPs, -ALIs, nodes, local end switches? Could he take the charts out -in the street and show them to anybody, "without violating -some proprietary notion that BellSouth has?" - -Ms Williams showed some confusion, but finally areed that the charts were, -in fact, public. - -"But isn't this what you said was basically what appeared in Phrack?" - -Ms. Williams denied this. - -Zenner now pointed out that the E911 Document as published in Phrack -was only half the size of the original E911 Document (as Prophet -had purloined it). Half of it had been deleted--edited by Neidorf. - -Ms. Williams countered that "Most of the information that is -in the text file is redundant." - -Zenner continued to probe. Exactly what bits of knowledge in the Document -were, in fact, unknown to the public? Locations of E911 computers? -Phone numbers for telco personnel? Ongoing maintenance subcommittees? -Hadn't Neidorf removed much of this? - -Then he pounced. "Are you familiar with Bellcore Technical Reference -Document TR-TSY-000350?" It was, Zenner explained, officially titled -"E911 Public Safety Answering Point Interface Between 1-1AESS Switch -and Customer Premises Equipment." It contained highly detailed -and specific technical information about the E911 System. -It was published by Bellcore and publicly available for about $20. - -He showed the witness a Bellcore catalog which listed thousands -of documents from Bellcore and from all the Baby Bells, BellSouth included. -The catalog, Zenner pointed out, was free. Anyone with a credit card -could call the Bellcore toll-free 800 number and simply order any -of these documents, which would be shipped to any customer without question. -Including, for instance, "BellSouth E911 Service Interfaces to -Customer Premises Equipment at a Public Safety Answering Point." - -Zenner gave the witness a copy of "BellSouth E911 Service Interfaces," -which cost, as he pointed out, $13, straight from the catalog. -"Look at it carefully," he urged Ms. Williams, "and tell me -if it doesn't contain about twice as much detailed information -about the E911 system of BellSouth than appeared anywhere in Phrack." - -"You want me to. . . ." Ms. Williams trailed off. "I don't understand." - -"Take a careful look," Zenner persisted. "Take a look at that document, -and tell me when you're done looking at it if, indeed, it doesn't contain -much more detailed information about the E911 system than appeared in Phrack." - -"Phrack wasn't taken from this," Ms. Williams said. - -"Excuse me?" said Zenner. - -"Phrack wasn't taken from this." - -"I can't hear you," Zenner said. - -"Phrack was not taken from this document. I don't understand -your question to me." - -"I guess you don't," Zenner said. - -At this point, the prosecution's case had been gutshot. -Ms. Williams was distressed. Her confusion was quite genuine. -Phrack had not been taken from any publicly available Bellcore document. -Phrack's E911 Document had been stolen from her own company's computers, -from her own company's text files, that her own colleagues had written, -and revised, with much labor. - -But the "value" of the Document had been blown to smithereens. -It wasn't worth eighty grand. According to Bellcore it was worth -thirteen bucks. And the looming menace that it supposedly posed -had been reduced in instants to a scarecrow. Bellcore itself -was selling material far more detailed and "dangerous," -to anybody with a credit card and a phone. - -Actually, Bellcore was not giving this information to just anybody. -They gave it to ANYBODY WHO ASKED, but not many did ask. -Not many people knew that Bellcore had a free catalog and an 800 number. -John Nagle knew, but certainly the average teenage phreak didn't know. -"Tuc," a friend of Neidorf's and sometime Phrack contributor, knew, -and Tuc had been very helpful to the defense, behind the scenes. -But the Legion of Doom didn't know--otherwise, they would never -have wasted so much time raiding dumpsters. Cook didn't know. -Foley didn't know. Kluepfel didn't know. The right hand -of Bellcore knew not what the left hand was doing. The right -hand was battering hackers without mercy, while the left hand -was distributing Bellcore's intellectual property to anybody -who was interested in telephone technical trivia--apparently, -a pathetic few. - -The digital underground was so amateurish and poorly organized -that they had never discovered this heap of unguarded riches. -The ivory tower of the telcos was so wrapped-up in the fog -of its own technical obscurity that it had left all the -windows open and flung open the doors. No one had even noticed. - -Zenner sank another nail in the coffin. He produced a printed issue -of Telephone Engineer & Management, a prominent industry journal -that comes out twice a month and costs $27 a year. This particular issue -of TE&M, called "Update on 911," featured a galaxy of technical details -on 911 service and a glossary far more extensive than Phrack's. - -The trial rumbled on, somehow, through its own momentum. -Tim Foley testified about his interrogations of Neidorf. -Neidorf's written admission that he had known the E911 Document -was pilfered was officially read into the court record. - -An interesting side issue came up: "Terminus" had once passed Neidorf -a piece of UNIX AT&T software, a log-in sequence, that had been cunningly -altered so that it could trap passwords. The UNIX software itself was -illegally copied AT&T property, and the alterations "Terminus" had made to it, -had transformed it into a device for facilitating computer break-ins. Terminus -himself would eventually plead guilty to theft of this piece of software, -and the Chicago group would send Terminus to prison for it. But it was -of dubious relevance in the Neidorf case. Neidorf hadn't written the program. -He wasn't accused of ever having used it. And Neidorf wasn't being charged -with software theft or owning a password trapper. - -On the next day, Zenner took the offensive. The civil libertarians -now had their own arcane, untried legal weaponry to launch into action-- -the Electronic Communications Privacy Act of 1986, 18 US Code, -Section 2701 et seq. Section 2701 makes it a crime to intentionally -access without authorization a facility in which an electronic communication -service is provided--it is, at heart, an anti-bugging and anti-tapping law, -intended to carry the traditional protections of telephones into other -electronic channels of communication. While providing penalties for amateur -snoops, however, Section 2703 of the ECPA also lays some formal difficulties -on the bugging and tapping activities of police. - -The Secret Service, in the person of Tim Foley, had served Richard Andrews -with a federal grand jury subpoena, in their pursuit of Prophet, -the E911 Document, and the Terminus software ring. But according to -the Electronic Communications Privacy Act, a "provider of remote -computing service" was legally entitled to "prior notice" from -the government if a subpoena was used. Richard Andrews and his -basement UNIX node, Jolnet, had not received any "prior notice." -Tim Foley had purportedly violated the ECPA and committed -an electronic crime! Zenner now sought the judge's permission -to cross-examine Foley on the topic of Foley's own electronic misdeeds. - -Cook argued that Richard Andrews' Jolnet was a privately owned -bulletin board, and not within the purview of ECPA. Judge Bua -granted the motion of the government to prevent cross-examination -on that point, and Zenner's offensive fizzled. This, however, -was the first direct assault on the legality of the actions -of the Computer Fraud and Abuse Task Force itself-- -the first suggestion that they themselves had broken the law, -and might, perhaps, be called to account. - -Zenner, in any case, did not really need the ECPA. -Instead, he grilled Foley on the glaring contradictions in -the supposed value of the E911 Document. He also brought up -the embarrassing fact that the supposedly red-hot E911 Document -had been sitting around for months, in Jolnet, with Kluepfel's knowledge, -while Kluepfel had done nothing about it. - -In the afternoon, the Prophet was brought in to testify -for the prosecution. (The Prophet, it will be recalled, -had also been indicted in the case as partner in a fraud -scheme with Neidorf.) In Atlanta, the Prophet had already -pled guilty to one charge of conspiracy, one charge of wire fraud -and one charge of interstate transportation of stolen property. -The wire fraud charge, and the stolen property charge, -were both directly based on the E911 Document. - -The twenty-year-old Prophet proved a sorry customer, -answering questions politely but in a barely audible mumble, -his voice trailing off at the ends of sentences. -He was constantly urged to speak up. - -Cook, examining Prophet, forced him to admit that -he had once had a "drug problem," abusing amphetamines, -marijuana, cocaine, and LSD. This may have established -to the jury that "hackers" are, or can be, seedy lowlife characters, -but it may have damaged Prophet's credibility somewhat. -Zenner later suggested that drugs might have damaged Prophet's memory. -The interesting fact also surfaced that Prophet had never -physically met Craig Neidorf. He didn't even know -Neidorf's last name--at least, not until the trial. - -Prophet confirmed the basic facts of his hacker career. -He was a member of the Legion of Doom. He had abused codes, -he had broken into switching stations and re-routed calls, -he had hung out on pirate bulletin boards. He had raided -the BellSouth AIMSX computer, copied the E911 Document, -stored it on Jolnet, mailed it to Neidorf. He and Neidorf -had edited it, and Neidorf had known where it came from. - -Zenner, however, had Prophet confirm that Neidorf was not a member -of the Legion of Doom, and had not urged Prophet to break into -BellSouth computers. Neidorf had never urged Prophet to defraud anyone, -or to steal anything. Prophet also admitted that he had never known Neidorf -to break in to any computer. Prophet said that no one in the Legion of Doom -considered Craig Neidorf a "hacker" at all. Neidorf was not a UNIX maven, -and simply lacked the necessary skill and ability to break into computers. -Neidorf just published a magazine. - -On Friday, July 27, 1990, the case against Neidorf collapsed. -Cook moved to dismiss the indictment, citing "information currently -available to us that was not available to us at the inception of the trial." -Judge Bua praised the prosecution for this action, which he described as -"very responsible," then dismissed a juror and declared a mistrial. - -Neidorf was a free man. His defense, however, had cost himself -and his family dearly. Months of his life had been consumed in anguish; -he had seen his closest friends shun him as a federal criminal. -He owed his lawyers over a hundred thousand dollars, despite -a generous payment to the defense by Mitch Kapor. - -Neidorf was not found innocent. The trial was simply dropped. -Nevertheless, on September 9, 1991, Judge Bua granted Neidorf's -motion for the "expungement and sealing" of his indictment record. -The United States Secret Service was ordered to delete and destroy -all fingerprints, photographs, and other records of arrest -or processing relating to Neidorf's indictment, including -their paper documents and their computer records. - -Neidorf went back to school, blazingly determined to become a lawyer. -Having seen the justice system at work, Neidorf lost much of his enthusiasm -for merely technical power. At this writing, Craig Neidorf is working -in Washington as a salaried researcher for the American Civil Liberties Union. - -# - -The outcome of the Neidorf trial changed the EFF -from voices-in-the-wilderness to the media darlings -of the new frontier. - -Legally speaking, the Neidorf case was not a sweeping triumph -for anyone concerned. No constitutional principles had been established. -The issues of "freedom of the press" for electronic publishers remained -in legal limbo. There were public misconceptions about the case. -Many people thought Neidorf had been found innocent and relieved -of all his legal debts by Kapor. The truth was that the government -had simply dropped the case, and Neidorf's family had gone deeply -into hock to support him. - -But the Neidorf case did provide a single, devastating, public sound-bite: -THE FEDS SAID IT WAS WORTH EIGHTY GRAND, AND IT WAS ONLY WORTH THIRTEEN BUCKS. - -This is the Neidorf case's single most memorable element. No serious report -of the case missed this particular element. Even cops could not read this -without a wince and a shake of the head. It left the public credibility -of the crackdown agents in tatters. - -The crackdown, in fact, continued, however. Those two charges -against Prophet, which had been based on the E911 Document, -were quietly forgotten at his sentencing--even though Prophet -had already pled guilty to them. Georgia federal prosecutors -strongly argued for jail time for the Atlanta Three, insisting on -"the need to send a message to the community," "the message that -hackers around the country need to hear." - -There was a great deal in their sentencing memorandum -about the awful things that various other hackers had done -(though the Atlanta Three themselves had not, in fact, -actually committed these crimes). There was also much -speculation about the awful things that the Atlanta Three -MIGHT have done and WERE CAPABLE of doing (even though -they had not, in fact, actually done them). -The prosecution's argument carried the day. -The Atlanta Three were sent to prison: -Urvile and Leftist both got 14 months each, -while Prophet (a second offender) got 21 months. - -The Atlanta Three were also assessed staggering fines as "restitution": -$233,000 each. BellSouth claimed that the defendants had "stolen" -"approximately $233,880 worth" of "proprietary computer access information"-- -specifically, $233,880 worth of computer passwords and connect addresses. -BellSouth's astonishing claim of the extreme value of its own computer -passwords and addresses was accepted at face value by the Georgia court. -Furthermore (as if to emphasize its theoretical nature) this enormous sum -was not divvied up among the Atlanta Three, but each of them had to pay -all of it. - -A striking aspect of the sentence was that the Atlanta Three were -specifically forbidden to use computers, except for work or under supervision. -Depriving hackers of home computers and modems makes some sense if one -considers hackers as "computer addicts," but EFF, filing an amicus brief -in the case, protested that this punishment was unconstitutional-- -it deprived the Atlanta Three of their rights of free association -and free expression through electronic media. - -Terminus, the "ultimate hacker," was finally sent to prison for a year -through the dogged efforts of the Chicago Task Force. His crime, -to which he pled guilty, was the transfer of the UNIX password trapper, -which was officially valued by AT&T at $77,000, a figure which aroused -intense skepticism among those familiar with UNIX "login.c" programs. - -The jailing of Terminus and the Atlanta Legionnaires of Doom, however, -did not cause the EFF any sense of embarrassment or defeat. -On the contrary, the civil libertarians were rapidly gathering strength. - -An early and potent supporter was Senator Patrick Leahy, -Democrat from Vermont, who had been a Senate sponsor -of the Electronic Communications Privacy Act. Even before -the Neidorf trial, Leahy had spoken out in defense of hacker-power -and freedom of the keyboard: "We cannot unduly inhibit the inquisitive -13-year-old who, if left to experiment today, may tomorrow develop -the telecommunications or computer technology to lead the United States -into the 21st century. He represents our future and our best hope -to remain a technologically competitive nation." - -It was a handsome statement, rendered perhaps rather more effective -by the fact that the crackdown raiders DID NOT HAVE any Senators -speaking out for THEM. On the contrary, their highly secretive -actions and tactics, all "sealed search warrants" here and -"confidential ongoing investigations" there, might have won -them a burst of glamorous publicity at first, but were crippling -them in the on-going propaganda war. Gail Thackeray was reduced -to unsupported bluster: "Some of these people who are loudest -on the bandwagon may just slink into the background," -she predicted in Newsweek--when all the facts came out, -and the cops were vindicated. - -But all the facts did not come out. Those facts that did, -were not very flattering. And the cops were not vindicated. -And Gail Thackeray lost her job. By the end of 1991, -William Cook had also left public employment. - -1990 had belonged to the crackdown, but by '91 its agents -were in severe disarray, and the libertarians were on a roll. -People were flocking to the cause. - -A particularly interesting ally had been Mike Godwin of Austin, Texas. -Godwin was an individual almost as difficult to describe as Barlow; -he had been editor of the student newspaper of the University of Texas, -and a computer salesman, and a programmer, and in 1990 was back -in law school, looking for a law degree. - -Godwin was also a bulletin board maven. He was very well-known -in the Austin board community under his handle "Johnny Mnemonic," -which he adopted from a cyberpunk science fiction story by William Gibson. -Godwin was an ardent cyberpunk science fiction fan. As a fellow Austinite -of similar age and similar interests, I myself had known Godwin socially -for many years. When William Gibson and myself had been writing our -collaborative SF novel, The Difference Engine, Godwin had been our -technical advisor in our effort to link our Apple word-processors -from Austin to Vancouver. Gibson and I were so pleased by his generous -expert help that we named a character in the novel "Michael Godwin" -in his honor. - -The handle "Mnemonic" suited Godwin very well. His erudition -and his mastery of trivia were impressive to the point of stupor; -his ardent curiosity seemed insatiable, and his desire to debate -and argue seemed the central drive of his life. Godwin had even -started his own Austin debating society, wryly known as the -"Dull Men's Club." In person, Godwin could be overwhelming; -a flypaper-brained polymath who could not seem to let any idea go. -On bulletin boards, however, Godwin's closely reasoned, -highly grammatical, erudite posts suited the medium well, -and he became a local board celebrity. - -Mike Godwin was the man most responsible for the public national exposure -of the Steve Jackson case. The Izenberg seizure in Austin had received -no press coverage at all. The March 1 raids on Mentor, Bloodaxe, and -Steve Jackson Games had received a brief front-page splash in the -front page of the Austin American-Statesman, but it was confused -and ill-informed: the warrants were sealed, and the Secret Service -wasn't talking. Steve Jackson seemed doomed to obscurity. -Jackson had not been arrested; he was not charged with any crime; -he was not on trial. He had lost some computers in an ongoing -investigation--so what? Jackson tried hard to attract attention -to the true extent of his plight, but he was drawing a blank; -no one in a position to help him seemed able to get a mental grip -on the issues. - -Godwin, however, was uniquely, almost magically, qualified -to carry Jackson's case to the outside world. Godwin was -a board enthusiast, a science fiction fan, a former journalist, -a computer salesman, a lawyer-to-be, and an Austinite. -Through a coincidence yet more amazing, in his last year -of law school Godwin had specialized in federal prosecutions -and criminal procedure. Acting entirely on his own, Godwin made -up a press packet which summarized the issues and provided useful -contacts for reporters. Godwin's behind-the-scenes effort -(which he carried out mostly to prove a point in a local board debate) -broke the story again in the Austin American-Statesman and then in Newsweek. - -Life was never the same for Mike Godwin after that. As he joined the growing -civil liberties debate on the Internet, it was obvious to all parties involved -that here was one guy who, in the midst of complete murk and confusion, -GENUINELY UNDERSTOOD EVERYTHING HE WAS TALKING ABOUT. The disparate elements -of Godwin's dilettantish existence suddenly fell together as neatly as -the facets of a Rubik's cube. - -When the time came to hire a full-time EFF staff attorney, -Godwin was the obvious choice. He took the Texas bar exam, -left Austin, moved to Cambridge, became a full-time, professional, -computer civil libertarian, and was soon touring the nation on behalf -of EFF, delivering well-received addresses on the issues to crowds -as disparate as academics, industrialists, science fiction fans, -and federal cops. - -Michael Godwin is currently the chief legal counsel of -the Electronic Frontier Foundation in Cambridge, Massachusetts. - -# - -Another early and influential participant in the controversy -was Dorothy Denning. Dr. Denning was unique among investigators -of the computer underground in that she did not enter the debate -with any set of politicized motives. She was a professional -cryptographer and computer security expert whose primary interest -in hackers was SCHOLARLY. She had a B.A. and M.A. in mathematics, -and a Ph.D. in computer science from Purdue. She had worked for SRI -International, the California think-tank that was also the home of -computer-security maven Donn Parker, and had authored an influential text -called Cryptography and Data Security. In 1990, Dr. Denning was working for -Digital Equipment Corporation in their Systems Reseach Center. Her husband, -Peter Denning, was also a computer security expert, working for NASA's -Research Institute for Advanced Computer Science. He had edited the -well-received Computers Under Attack: Intruders, Worms and Viruses. - -Dr. Denning took it upon herself to contact the digital underground, -more or less with an anthropological interest. There she discovered -that these computer-intruding hackers, who had been characterized -as unethical, irresponsible, and a serious danger to society, -did in fact have their own subculture and their own rules. -They were not particularly well-considered rules, but they were, -in fact, rules. Basically, they didn't take money and they -didn't break anything. - -Her dispassionate reports on her researches did a great deal -to influence serious-minded computer professionals--the sort -of people who merely rolled their eyes at the cyberspace -rhapsodies of a John Perry Barlow. - -For young hackers of the digital underground, meeting Dorothy Denning -was a genuinely mind-boggling experience. Here was this neatly coiffed, -conservatively dressed, dainty little personage, who reminded most -hackers of their moms or their aunts. And yet she was an IBM systems -programmer with profound expertise in computer architectures -and high-security information flow, who had personal friends -in the FBI and the National Security Agency. - -Dorothy Denning was a shining example of the American mathematical -intelligentsia, a genuinely brilliant person from the central ranks -of the computer-science elite. And here she was, gently questioning -twenty-year-old hairy-eyed phone-phreaks over the deeper ethical -implications of their behavior. - -Confronted by this genuinely nice lady, most hackers sat up very straight -and did their best to keep the anarchy-file stuff down to a faint whiff -of brimstone. Nevertheless, the hackers WERE in fact prepared to seriously -discuss serious issues with Dorothy Denning. They were willing to speak -the unspeakable and defend the indefensible, to blurt out their convictions -that information cannot be owned, that the databases of governments and large -corporations were a threat to the rights and privacy of individuals. - -Denning's articles made it clear to many that "hacking" -was not simple vandalism by some evil clique of psychotics. -"Hacking" was not an aberrant menace that could be charmed away -by ignoring it, or swept out of existence by jailing a few ringleaders. -Instead, "hacking" was symptomatic of a growing, primal struggle over -knowledge and power in the age of information. - -Denning pointed out that the attitude of hackers were at least partially -shared by forward-looking management theorists in the business community: -people like Peter Drucker and Tom Peters. Peter Drucker, in his book -The New Realities, had stated that "control of information by the government -is no longer possible. Indeed, information is now transnational. -Like money, it has no `fatherland.'" - -And management maven Tom Peters had chided large corporations for uptight, -proprietary attitudes in his bestseller, Thriving on Chaos: -"Information hoarding, especially by politically motivated, -power-seeking staffs, had been commonplace throughout American industry, -service and manufacturing alike. It will be an impossible -millstone aroung the neck of tomorrow's organizations." - -Dorothy Denning had shattered the social membrane of the -digital underground. She attended the Neidorf trial, -where she was prepared to testify for the defense as an expert witness. -She was a behind-the-scenes organizer of two of the most important -national meetings of the computer civil libertarians. Though not -a zealot of any description, she brought disparate elements of the -electronic community into a surprising and fruitful collusion. - -Dorothy Denning is currently the Chair of the Computer Science Department -at Georgetown University in Washington, DC. - -# - -There were many stellar figures in the civil libertarian community. -There's no question, however, that its single most influential figure -was Mitchell D. Kapor. Other people might have formal titles, -or governmental positions, have more experience with crime, -or with the law, or with the arcanities of computer security -or constitutional theory. But by 1991 Kapor had transcended -any such narrow role. Kapor had become "Mitch." - -Mitch had become the central civil-libertarian ad-hocrat. -Mitch had stood up first, he had spoken out loudly, directly, -vigorously and angrily, he had put his own reputation, -and his very considerable personal fortune, on the line. -By mid-'91 Kapor was the best-known advocate of his cause -and was known PERSONALLY by almost every single human being in America -with any direct influence on the question of civil liberties in cyberspace. -Mitch had built bridges, crossed voids, changed paradigms, forged metaphors, -made phone-calls and swapped business cards to such spectacular effect -that it had become impossible for anyone to take any action in the -"hacker question" without wondering what Mitch might think-- -and say--and tell his friends. - -The EFF had simply NETWORKED the situation into an entirely new status quo. -And in fact this had been EFF's deliberate strategy from the beginning. -Both Barlow and Kapor loathed bureaucracies and had deliberately -chosen to work almost entirely through the electronic spiderweb of -"valuable personal contacts." - -After a year of EFF, both Barlow and Kapor had every reason -to look back with satisfaction. EFF had established its own Internet node, -"eff.org," with a well-stocked electronic archive of documents on -electronic civil rights, privacy issues, and academic freedom. -EFF was also publishing EFFector, a quarterly printed journal, -as well as EFFector Online, an electronic newsletter with -over 1,200 subscribers. And EFF was thriving on the Well. - -EFF had a national headquarters in Cambridge and a full-time staff. -It had become a membership organization and was attracting -grass-roots support. It had also attracted the support -of some thirty civil-rights lawyers, ready and eager -to do pro bono work in defense of the Constitution in Cyberspace. - -EFF had lobbied successfully in Washington and in Massachusetts -to change state and federal legislation on computer networking. -Kapor in particular had become a veteran expert witness, -and had joined the Computer Science and Telecommunications Board -of the National Academy of Science and Engineering. - -EFF had sponsored meetings such as "Computers, Freedom and Privacy" -and the CPSR Roundtable. It had carried out a press offensive that, -in the words of EFFector, "has affected the climate of opinion about -computer networking and begun to reverse the slide into -`hacker hysteria' that was beginning to grip the nation." - -It had helped Craig Neidorf avoid prison. - -And, last but certainly not least, the Electronic Frontier Foundation -had filed a federal lawsuit in the name of Steve Jackson, -Steve Jackson Games Inc., and three users of the Illuminati -bulletin board system. The defendants were, and are, -the United States Secret Service, William Cook, Tim Foley, -Barbara Golden and Henry Kleupfel. - -The case, which is in pre-trial procedures in an Austin federal court -as of this writing, is a civil action for damages to redress -alleged violations of the First and Fourth Amendments to the -United States Constitution, as well as the Privacy Protection Act -of 1980 (42 USC 2000aa et seq.), and the Electronic Communications -Privacy Act (18 USC 2510 et seq and 2701 et seq). - -EFF had established that it had credibility. It had also established -that it had teeth. - -In the fall of 1991 I travelled to Massachusetts to speak personally -with Mitch Kapor. It was my final interview for this book. - -# - -The city of Boston has always been one of the major intellectual centers -of the American republic. It is a very old city by American standards, -a place of skyscrapers overshadowing seventeenth-century graveyards, -where the high-tech start-up companies of Route 128 co-exist with the -hand-wrought pre-industrial grace of "Old Ironsides," the USS CONSTITUTION. - -The Battle of Bunker Hill, one of the first and bitterest armed clashes -of the American Revolution, was fought in Boston's environs. Today there is -a monumental spire on Bunker Hill, visible throughout much of the city. -The willingness of the republican revolutionaries to take up arms and fire -on their oppressors has left a cultural legacy that two full centuries -have not effaced. Bunker Hill is still a potent center of American political -symbolism, and the Spirit of '76 is still a potent image for those who seek -to mold public opinion. - -Of course, not everyone who wraps himself in the flag is necessarily -a patriot. When I visited the spire in September 1991, it bore a huge, -badly-erased, spray-can grafitto around its bottom reading -"BRITS OUT--IRA PROVOS." Inside this hallowed edifice was -a glass-cased diorama of thousands of tiny toy soldiers, -rebels and redcoats, fighting and dying over the green hill, -the riverside marshes, the rebel trenchworks. Plaques indicated the -movement of troops, the shiftings of strategy. The Bunker Hill Monument -is occupied at its very center by the toy soldiers of a military -war-game simulation. - -The Boston metroplex is a place of great universities, -prominent among the Massachusetts Institute of Technology, -where the term "computer hacker" was first coined. The Hacker Crackdown -of 1990 might be interpreted as a political struggle among American cities: -traditional strongholds of longhair intellectual liberalism, -such as Boston, San Francisco, and Austin, versus the bare-knuckle -industrial pragmatism of Chicago and Phoenix (with Atlanta and New York -wrapped in internal struggle). - -The headquarters of the Electronic Frontier Foundation is on -155 Second Street in Cambridge, a Bostonian suburb north -of the River Charles. Second Street has weedy sidewalks of dented, -sagging brick and elderly cracked asphalt; large street-signs warn -"NO PARKING DURING DECLARED SNOW EMERGENCY." This is an old area -of modest manufacturing industries; the EFF is catecorner from the -Greene Rubber Company. EFF's building is two stories of red brick; -its large wooden windows feature gracefully arched tops and stone sills. - -The glass window beside the Second Street entrance bears three sheets -of neatly laser-printed paper, taped against the glass. They read: -ON Technology. EFF. KEI. - -"ON Technology" is Kapor's software company, which currently specializes -in "groupware" for the Apple Macintosh computer. "Groupware" is intended -to promote efficient social interaction among office-workers linked -by computers. ON Technology's most successful software products to date -are "Meeting Maker" and "Instant Update." - -"KEI" is Kapor Enterprises Inc., Kapor's personal holding company, -the commercial entity that formally controls his extensive investments -in other hardware and software corporations. - -"EFF" is a political action group--of a special sort. - -Inside, someone's bike has been chained to the handrails -of a modest flight of stairs. A wall of modish glass brick -separates this anteroom from the offices. Beyond the brick, -there's an alarm system mounted on the wall, a sleek, complex little -number that resembles a cross between a thermostat and a CD player. -Piled against the wall are box after box of a recent special issue -of Scientific American, "How to Work, Play, and Thrive in Cyberspace," -with extensive coverage of electronic networking techniques -and political issues, including an article by Kapor himself. -These boxes are addressed to Gerard Van der Leun, EFF's -Director of Communications, who will shortly mail those magazines -to every member of the EFF. - -The joint headquarters of EFF, KEI, and ON Technology, -which Kapor currently rents, is a modestly bustling place. -It's very much the same physical size as Steve Jackson's gaming company. -It's certainly a far cry from the gigantic gray steel-sided railway -shipping barn, on the Monsignor O'Brien Highway, that is owned -by Lotus Development Corporation. - -Lotus is, of course, the software giant that Mitchell Kapor founded -in the late 70s. The software program Kapor co-authored, -"Lotus 1-2-3," is still that company's most profitable product. -"Lotus 1-2-3" also bears a singular distinction in the -digital underground: it's probably the most pirated piece -of application software in world history. - -Kapor greets me cordially in his own office, down a hall. -Kapor, whose name is pronounced KAY-por, is in his early forties, -married and the father of two. He has a round face, high forehead, -straight nose, a slightly tousled mop of black hair peppered with gray. -His large brown eyes are wideset, reflective, one might almost say soulful. -He disdains ties, and commonly wears Hawaiian shirts and tropical prints, -not so much garish as simply cheerful and just that little bit anomalous. - -There is just the whiff of hacker brimstone about Mitch Kapor. -He may not have the hard-riding, hell-for-leather, guitar-strumming -charisma of his Wyoming colleague John Perry Barlow, but there's -something about the guy that still stops one short. He has the air -of the Eastern city dude in the bowler hat, the dreamy, -Longfellow-quoting poker shark who only HAPPENS to know -the exact mathematical odds against drawing to an inside straight. -Even among his computer-community colleagues, who are hardly known -for mental sluggishness, Kapor strikes one forcefully as a very -intelligent man. He speaks rapidly, with vigorous gestures, -his Boston accent sometimes slipping to the sharp nasal tang -of his youth in Long Island. - -Kapor, whose Kapor Family Foundation does much of his philanthropic work, -is a strong supporter of Boston's Computer Museum. Kapor's interest -in the history of his industry has brought him some remarkable curios, -such as the "byte" just outside his office door. This "byte"-- -eight digital bits--has been salvaged from the wreck of an -electronic computer of the pre-transistor age. It's a standing gunmetal -rack about the size of a small toaster-oven: with eight slots -of hand-soldered breadboarding featuring thumb-sized vacuum tubes. -If it fell off a table it could easily break your foot, -but it was state-of-the-art computation in the 1940s. -(It would take exactly 157,184 of these primordial toasters -to hold the first part of this book.) - -There's also a coiling, multicolored, scaly dragon that some -inspired techno-punk artist has cobbled up entirely out of transistors, -capacitors, and brightly plastic-coated wiring. - -Inside the office, Kapor excuses himself briefly to do a little -mouse-whizzing housekeeping on his personal Macintosh IIfx. -If its giant screen were an open window, an agile person -could climb through it without much trouble at all. -There's a coffee-cup at Kapor's elbow, a memento of his -recent trip to Eastern Europe, which has a black-and-white -stencilled photo and the legend CAPITALIST FOOLS TOUR. -It's Kapor, Barlow, and two California venture-capitalist luminaries -of their acquaintance, four windblown, grinning Baby Boomer -dudes in leather jackets, boots, denim, travel bags, -standing on airport tarmac somewhere behind the formerly Iron Curtain. -They look as if they're having the absolute time of their lives. - -Kapor is in a reminiscent mood. We talk a bit about his youth-- -high school days as a "math nerd," Saturdays attending Columbia University's -high-school science honors program, where he had his first experience -programming computers. IBM 1620s, in 1965 and '66. "I was very interested," -says Kapor, "and then I went off to college and got distracted by drugs sex -and rock and roll, like anybody with half a brain would have then!" -After college he was a progressive-rock DJ in Hartford, Connecticut, -for a couple of years. - -I ask him if he ever misses his rock and roll days--if he ever wished -he could go back to radio work. - -He shakes his head flatly. "I stopped thinking about going back -to be a DJ the day after Altamont." - -Kapor moved to Boston in 1974 and got a job programming mainframes in COBOL. -He hated it. He quit and became a teacher of transcendental meditation. -(It was Kapor's long flirtation with Eastern mysticism that gave the -world "Lotus.") - -In 1976 Kapor went to Switzerland, where the Transcendental Meditation -movement had rented a gigantic Victorian hotel in St-Moritz. It was -an all-male group--a hundred and twenty of them--determined upon -Enlightenment or Bust. Kapor had given the transcendant his best shot. -He was becoming disenchanted by "the nuttiness in the organization." -"They were teaching people to levitate," he says, staring at the floor. -His voice drops an octave, becomes flat. "THEY DON'T LEVITATE." - -Kapor chose Bust. He went back to the States and acquired a degree -in counselling psychology. He worked a while in a hospital, -couldn't stand that either. "My rep was," he says "a very bright kid -with a lot of potential who hasn't found himself. Almost thirty. -Sort of lost." - -Kapor was unemployed when he bought his first personal computer--an Apple II. -He sold his stereo to raise cash and drove to New Hampshire to avoid the -sales tax. - -"The day after I purchased it," Kapor tells me, "I was hanging out -in a computer store and I saw another guy, a man in his forties, -well-dressed guy, and eavesdropped on his conversation with the salesman. -He didn't know anything about computers. I'd had a year programming. -And I could program in BASIC. I'd taught myself. So I went up to him, -and I actually sold myself to him as a consultant." He pauses. -"I don't know where I got the nerve to do this. It was uncharacteristic. -I just said, `I think I can help you, I've been listening, -this is what you need to do and I think I can do it for you.' -And he took me on! He was my first client! I became a computer -consultant the first day after I bought the Apple II." - -Kapor had found his true vocation. He attracted more clients -for his consultant service, and started an Apple users' group. - -A friend of Kapor's, Eric Rosenfeld, a graduate student at MIT, -had a problem. He was doing a thesis on an arcane form of -financial statistics, but could not wedge himself into the crowded queue -for time on MIT's mainframes. (One might note at this point that if -Mr. Rosenfeld had dishonestly broken into the MIT mainframes, -Kapor himself might have never invented Lotus 1-2-3 and -the PC business might have been set back for years!) -Eric Rosenfeld did have an Apple II, however, -and he thought it might be possible to scale the problem down. -Kapor, as favor, wrote a program for him in BASIC that did the job. - -It then occurred to the two of them, out of the blue, -that it might be possible to SELL this program. -They marketed it themselves, in plastic baggies, -for about a hundred bucks a pop, mail order. -"This was a total cottage industry by a marginal consultant," -Kapor says proudly. "That's how I got started, honest to God." - -Rosenfeld, who later became a very prominent figure on Wall Street, -urged Kapor to go to MIT's business school for an MBA. -Kapor did seven months there, but never got his MBA. -He picked up some useful tools--mainly a firm grasp -of the principles of accounting--and, in his own words, -"learned to talk MBA." Then he dropped out and went to Silicon Valley. - -The inventors of VisiCalc, the Apple computer's premier business program, -had shown an interest in Mitch Kapor. Kapor worked diligently for them -for six months, got tired of California, and went back to Boston -where they had better bookstores. The VisiCalc group had made -the critical error of bringing in "professional management." -"That drove them into the ground," Kapor says. - -"Yeah, you don't hear a lot about VisiCalc these days," I muse. - -Kapor looks surprised. "Well, Lotus. . . we BOUGHT it." - -"Oh. You BOUGHT it?" - -"Yeah." - -"Sort of like the Bell System buying Western Union?" - -Kapor grins. "Yep! Yep! Yeah, exactly!" - -Mitch Kapor was not in full command of the destiny of himself -or his industry. The hottest software commodities of the early 1980s -were COMPUTER GAMES--the Atari seemed destined to enter every teenage home -in America. Kapor got into business software simply because he didn't have -any particular feeling for computer games. But he was supremely fast -on his feet, open to new ideas and inclined to trust his instincts. -And his instincts were good. He chose good people to deal with-- -gifted programmer Jonathan Sachs (the co-author of Lotus 1-2-3). -Financial wizard Eric Rosenfeld, canny Wall Street analyst -and venture capitalist Ben Rosen. Kapor was the founder and CEO of Lotus, -one of the most spectacularly successful business ventures of the -later twentieth century. - -He is now an extremely wealthy man. I ask him if he actually -knows how much money he has. - -"Yeah," he says. "Within a percent or two." - -How much does he actually have, then? - -He shakes his head. "A lot. A lot. Not something I talk about. -Issues of money and class are things that cut pretty close to the bone." - -I don't pry. It's beside the point. One might presume, impolitely, -that Kapor has at least forty million--that's what he got the year -he left Lotus. People who ought to know claim Kapor has about -a hundred and fifty million, give or take a market swing -in his stock holdings. If Kapor had stuck with Lotus, -as his colleague friend and rival Bill Gates has stuck -with his own software start-up, Microsoft, then Kapor -would likely have much the same fortune Gates has-- -somewhere in the neighborhood of three billion, -give or take a few hundred million. Mitch Kapor -has all the money he wants. Money has lost whatever charm -it ever held for him--probably not much in the first place. -When Lotus became too uptight, too bureaucratic, too far -from the true sources of his own satisfaction, Kapor walked. -He simply severed all connections with the company and went out the door. -It stunned everyone--except those who knew him best. - -Kapor has not had to strain his resources to wreak a thorough -transformation in cyberspace politics. In its first year, -EFF's budget was about a quarter of a million dollars. -Kapor is running EFF out of his pocket change. - -Kapor takes pains to tell me that he does not consider himself -a civil libertarian per se. He has spent quite some time -with true-blue civil libertarians lately, and there's a -political-correctness to them that bugs him. They seem -to him to spend entirely too much time in legal nitpicking -and not enough vigorously exercising civil rights in the -everyday real world. - -Kapor is an entrepreneur. Like all hackers, he prefers his involvements -direct, personal, and hands-on. "The fact that EFF has a node on the -Internet is a great thing. We're a publisher. We're a distributor -of information." Among the items the eff.org Internet node carries -is back issues of Phrack. They had an internal debate about that in EFF, -and finally decided to take the plunge. They might carry other -digital underground publications--but if they do, he says, -"we'll certainly carry Donn Parker, and anything Gail Thackeray -wants to put up. We'll turn it into a public library, that has -the whole spectrum of use. Evolve in the direction of people making up -their own minds." He grins. "We'll try to label all the editorials." - -Kapor is determined to tackle the technicalities of the Internet -in the service of the public interest. "The problem with being a node -on the Net today is that you've got to have a captive technical specialist. -We have Chris Davis around, for the care and feeding of the balky beast! -We couldn't do it ourselves!" - -He pauses. "So one direction in which technology has to evolve -is much more standardized units, that a non-technical person -can feel comfortable with. It's the same shift as from minicomputers to PCs. -I can see a future in which any person can have a Node on the Net. -Any person can be a publisher. It's better than the media we now have. -It's possible. We're working actively." - -Kapor is in his element now, fluent, thoroughly in command in his material. -"You go tell a hardware Internet hacker that everyone should have a node -on the Net," he says, "and the first thing they're going to say is, -`IP doesn't scale!'" ("IP" is the interface protocol for the Internet. -As it currently exists, the IP software is simply not capable of -indefinite expansion; it will run out of usable addresses, it will saturate.) -"The answer," Kapor says, "is: evolve the protocol! Get the smart people -together and figure out what to do. Do we add ID? Do we add new protocol? -Don't just say, WE CAN'T DO IT." - -Getting smart people together to figure out what to do is a skill -at which Kapor clearly excels. I counter that people on the Internet -rather enjoy their elite technical status, and don't seem particularly -anxious to democratize the Net. - -Kapor agrees, with a show of scorn. "I tell them that this is the snobbery -of the people on the Mayflower looking down their noses at the people -who came over ON THE SECOND BOAT! Just because they got here a year, -or five years, or ten years before everybody else, that doesn't give -them ownership of cyberspace! By what right?" - -I remark that the telcos are an electronic network, too, -and they seem to guard their specialized knowledge pretty closely. - -Kapor ripostes that the telcos and the Internet are entirely -different animals. "The Internet is an open system, -everything is published, everything gets argued about, -basically by anybody who can get in. Mostly, it's exclusive -and elitist just because it's so difficult. Let's make it easier to use." - -On the other hand, he allows with a swift change of emphasis, -the so-called elitists do have a point as well. "Before people start coming in, -who are new, who want to make suggestions, and criticize the Net as -`all screwed up'. . . . They should at least take the time to understand -the culture on its own terms. It has its own history--show some respect -for it. I'm a conservative, to that extent." - -The Internet is Kapor's paradigm for the future of telecommunications. -The Internet is decentralized, non-hierarchical, almost anarchic. -There are no bosses, no chain of command, no secret data. -If each node obeys the general interface standards, -there's simply no need for any central network authority. - -Wouldn't that spell the doom of AT&T as an institution? I ask. - -That prospect doesn't faze Kapor for a moment. "Their big advantage, -that they have now, is that they have all of the wiring. -But two things are happening. Anyone with right-of-way -is putting down fiber--Southern Pacific Railroad, -people like that--there's enormous `dark fiber' laid in." -("Dark Fiber" is fiber-optic cable, whose enormous capacity -so exceeds the demands of current usage that much of the -fiber still has no light-signals on it--it's still `dark,' -awaiting future use.) - -"The other thing that's happening is the local-loop stuff -is going to go wireless. Everyone from Bellcore to the cable TV -companies to AT&T wants to put in these things called -`personal communication systems.' So you could have local competition-- -you could have multiplicity of people, a bunch of neighborhoods, -sticking stuff up on poles. And a bunch of other people laying in dark fiber. -So what happens to the telephone companies? There's enormous pressure -on them from both sides. - -"The more I look at this, the more I believe that in a post-industrial, -digital world, the idea of regulated monopolies is bad. People will -look back on it and say that in the 19th and 20th centuries -the idea of public utilities was an okay compromise. -You needed one set of wires in the ground. It was too economically -inefficient, otherwise. And that meant one entity running it. -But now, with pieces being wireless--the connections are going -to be via high-level interfaces, not via wires. I mean, ULTIMATELY -there are going to be wires--but the wires are just a commodity. -Fiber, wireless. You no longer NEED a utility." - -Water utilities? Gas utilities? - -Of course we still need those, he agrees. "But when what you're moving -is information, instead of physical substances, then you can play by -a different set of rules. We're evolving those rules now! -Hopefully you can have a much more decentralized system, -and one in which there's more competition in the marketplace. - -"The role of government will be to make sure that nobody cheats. -The proverbial `level playing field.' A policy that prevents monopolization. -It should result in better service, lower prices, more choices, -and local empowerment." He smiles. "I'm very big on local empowerment." - -Kapor is a man with a vision. It's a very novel vision which he -and his allies are working out in considerable detail and with great energy. -Dark, cynical, morbid cyberpunk that I am, I cannot avoid considering -some of the darker implications of "decentralized, nonhierarchical, -locally empowered" networking. - -I remark that some pundits have suggested that electronic networking--faxes, -phones, small-scale photocopiers--played a strong role in dissolving -the power of centralized communism and causing the collapse of the Warsaw Pact. - -Socialism is totally discredited, says Kapor, fresh back from -the Eastern Bloc. The idea that faxes did it, all by themselves, -is rather wishful thinking. - -Has it occurred to him that electronic networking might corrode -America's industrial and political infrastructure to the point -where the whole thing becomes untenable, unworkable--and the old order -just collapses headlong, like in Eastern Europe? - -"No," Kapor says flatly. "I think that's extraordinarily unlikely. -In part, because ten or fifteen years ago, I had similar hopes -about personal computers--which utterly failed to materialize." -He grins wryly, then his eyes narrow. "I'm VERY opposed to techno-utopias. -Every time I see one, I either run away, or try to kill it." - -It dawns on me then that Mitch Kapor is not trying to -make the world safe for democracy. He certainly is not -trying to make it safe for anarchists or utopians-- -least of all for computer intruders or electronic rip-off artists. -What he really hopes to do is make the world safe for -future Mitch Kapors. This world of decentralized, small-scale nodes, -with instant global access for the best and brightest, -would be a perfect milieu for the shoestring attic capitalism -that made Mitch Kapor what he is today. - -Kapor is a very bright man. He has a rare combination -of visionary intensity with a strong practical streak. -The Board of the EFF: John Barlow, Jerry Berman of the ACLU, -Stewart Brand, John Gilmore, Steve Wozniak, and Esther Dyson, -the doyenne of East-West computer entrepreneurism--share his gift, -his vision, and his formidable networking talents. -They are people of the 1960s, winnowed-out by its turbulence -and rewarded with wealth and influence. They are some of the best -and the brightest that the electronic community has to offer. -But can they do it, in the real world? Or are they only dreaming? -They are so few. And there is so much against them. - -I leave Kapor and his networking employees struggling cheerfully -with the promising intricacies of their newly installed Macintosh -System 7 software. The next day is Saturday. EFF is closed. -I pay a few visits to points of interest downtown. - -One of them is the birthplace of the telephone. - -It's marked by a bronze plaque in a plinth of black-and-white speckled granite. It sits in the -plaza of the John F. Kennedy Federal Building, the very place where Kapor was -once fingerprinted by the FBI. - -The plaque has a bas-relief picture of Bell's original telephone. -"BIRTHPLACE OF THE TELEPHONE," it reads. "Here, on June 2, 1875, -Alexander Graham Bell and Thomas A. Watson first transmitted sound over wires. - -"This successful experiment was completed in a fifth floor garret -at what was then 109 Court Street and marked the beginning of -world-wide telephone service." - -109 Court Street is long gone. Within sight of Bell's plaque, -across a street, is one of the central offices of NYNEX, -the local Bell RBOC, on 6 Bowdoin Square. - -I cross the street and circle the telco building, slowly, -hands in my jacket pockets. It's a bright, windy, New England -autumn day. The central office is a handsome 1940s-era megalith -in late Art Deco, eight stories high. - -Parked outside the back is a power-generation truck. -The generator strikes me as rather anomalous. Don't they -already have their own generators in this eight-story monster? -Then the suspicion strikes me that NYNEX must have heard -of the September 17 AT&T power-outage which crashed New York City. -Belt-and-suspenders, this generator. Very telco. - -Over the glass doors of the front entrance is a handsome bronze -bas-relief of Art Deco vines, sunflowers, and birds, entwining -the Bell logo and the legend NEW ENGLAND TELEPHONE AND TELEGRAPH COMPANY ---an entity which no longer officially exists. - -The doors are locked securely. I peer through the shadowed glass. -Inside is an official poster reading: - - -"New England Telephone a NYNEX Company - -ATTENTION - -"All persons while on New England Telephone -Company premises are required to visibly wear their -identification cards (C.C.P. Section 2, Page 1). - -"Visitors, vendors, contractors, and all others are -required to visibly wear a daily pass. - -"Thank you. - -Kevin C. Stanton. -Building Security Coordinator." - - -Outside, around the corner, is a pull-down ribbed metal security door, -a locked delivery entrance. Some passing stranger has grafitti-tagged -this door, with a single word in red spray-painted cursive: - -Fury - -# - -My book on the Hacker Crackdown is almost over now. -I have deliberately saved the best for last. - -In February 1991, I attended the CPSR Public Policy Roundtable, -in Washington, DC. CPSR, Computer Professionals for Social Responsibility, -was a sister organization of EFF, or perhaps its aunt, being older -and perhaps somewhat wiser in the ways of the world of politics. - -Computer Professionals for Social Responsibility began in 1981 -in Palo Alto, as an informal discussion group of Californian -computer scientists and technicians, united by nothing more -than an electronic mailing list. This typical high-tech -ad-hocracy received the dignity of its own acronym in 1982, -and was formally incorporated in 1983. - -CPSR lobbied government and public alike with an educational -outreach effort, sternly warning against any foolish -and unthinking trust in complex computer systems. -CPSR insisted that mere computers should never be -considered a magic panacea for humanity's social, -ethical or political problems. CPSR members were especially -troubled about the stability, safety, and dependability -of military computer systems, and very especially troubled -by those systems controlling nuclear arsenals. CPSR was -best-known for its persistent and well-publicized attacks on the -scientific credibility of the Strategic Defense Initiative ("Star Wars"). - -In 1990, CPSR was the nation's veteran cyber-political activist group, -with over two thousand members in twenty- one local chapters across the US. -It was especially active in Boston, Silicon Valley, and Washington DC, -where its Washington office sponsored the Public Policy Roundtable. - -The Roundtable, however, had been funded by EFF, which had passed CPSR -an extensive grant for operations. This was the first large-scale, -official meeting of what was to become the electronic civil -libertarian community. - -Sixty people attended, myself included--in this instance, not so much -as a journalist as a cyberpunk author. Many of the luminaries -of the field took part: Kapor and Godwin as a matter of course. -Richard Civille and Marc Rotenberg of CPSR. Jerry Berman of the ACLU. -John Quarterman, author of The Matrix. Steven Levy, author of Hackers. -George Perry and Sandy Weiss of Prodigy Services, there to network -about the civil-liberties troubles their young commercial -network was experiencing. Dr. Dorothy Denning. Cliff Figallo, -manager of the Well. Steve Jackson was there, having finally -found his ideal target audience, and so was Craig Neidorf, -"Knight Lightning" himself, with his attorney, Sheldon Zenner. -Katie Hafner, science journalist, and co-author of Cyberpunk: -Outlaws and Hackers on the Computer Frontier. Dave Farber, -ARPAnet pioneer and fabled Internet guru. Janlori Goldman -of the ACLU's Project on Privacy and Technology. John Nagle -of Autodesk and the Well. Don Goldberg of the House Judiciary Committee. -Tom Guidoboni, the defense attorney in the Internet Worm case. -Lance Hoffman, computer-science professor at The George Washington -University. Eli Noam of Columbia. And a host of others no less distinguished. - -Senator Patrick Leahy delivered the keynote address, -expressing his determination to keep ahead of the curve -on the issue of electronic free speech. The address was -well-received, and the sense of excitement was palpable. -Every panel discussion was interesting--some were entirely -compelling. People networked with an almost frantic interest. - -I myself had a most interesting and cordial lunch discussion with -Noel and Jeanne Gayler, Admiral Gayler being a former director -of the National Security Agency. As this was the first known encounter -between an actual no-kidding cyberpunk and a chief executive of -America's largest and best-financed electronic espionage apparat, -there was naturally a bit of eyebrow-raising on both sides. - -Unfortunately, our discussion was off-the-record. In fact -all the discussions at the CPSR were officially off-the-record, -the idea being to do some serious networking in an atmosphere -of complete frankness, rather than to stage a media circus. - -In any case, CPSR Roundtable, though interesting and intensely valuable, -was as nothing compared to the truly mind-boggling event that transpired -a mere month later. - -# - -"Computers, Freedom and Privacy." Four hundred people from -every conceivable corner of America's electronic community. -As a science fiction writer, I have been to some weird gigs in my day, -but this thing is truly BEYOND THE PALE. Even "Cyberthon," -Point Foundation's "Woodstock of Cyberspace" where Bay Area -psychedelia collided headlong with the emergent world -of computerized virtual reality, was like a Kiwanis Club gig -compared to this astonishing do. - -The "electronic community" had reached an apogee. -Almost every principal in this book is in attendance. -Civil Libertarians. Computer Cops. The Digital Underground. -Even a few discreet telco people. Colorcoded dots -for lapel tags are distributed. Free Expression issues. -Law Enforcement. Computer Security. Privacy. Journalists. -Lawyers. Educators. Librarians. Programmers. -Stylish punk-black dots for the hackers and phone phreaks. -Almost everyone here seems to wear eight or nine dots, -to have six or seven professional hats. - -It is a community. Something like Lebanon perhaps, -but a digital nation. People who had feuded all year -in the national press, people who entertained the deepest -suspicions of one another's motives and ethics, are now -in each others' laps. "Computers, Freedom and Privacy" -had every reason in the world to turn ugly, and yet except -for small irruptions of puzzling nonsense from the -convention's token lunatic, a surprising bonhomie reigned. -CFP was like a wedding-party in which two lovers, -unstable bride and charlatan groom, tie the knot -in a clearly disastrous matrimony. - -It is clear to both families--even to neighbors and random guests-- -that this is not a workable relationship, and yet the young couple's -desperate attraction can brook no further delay. They simply cannot -help themselves. Crockery will fly, shrieks from their newlywed home -will wake the city block, divorce waits in the wings like a vulture -over the Kalahari, and yet this is a wedding, and there is going -to be a child from it. Tragedies end in death; comedies in marriage. -The Hacker Crackdown is ending in marriage. And there will be a child. - -From the beginning, anomalies reign. John Perry Barlow, -cyberspace ranger, is here. His color photo in -The New York Times Magazine, Barlow scowling -in a grim Wyoming snowscape, with long black coat, -dark hat, a Macintosh SE30 propped on a fencepost -and an awesome frontier rifle tucked under one arm, -will be the single most striking visual image -of the Hacker Crackdown. And he is CFP's guest of honor-- -along with Gail Thackeray of the FCIC! What on earth do -they expect these dual guests to do with each other? Waltz? - -Barlow delivers the first address. Uncharacteristically, -he is hoarse--the sheer volume of roadwork has worn him down. -He speaks briefly, congenially, in a plea for conciliation, -and takes his leave to a storm of applause. - -Then Gail Thackeray takes the stage. She's visibly nervous. -She's been on the Well a lot lately. Reading those Barlow posts. -Following Barlow is a challenge to anyone. In honor of the famous -lyricist for the Grateful Dead, she announces reedily, she is going to read-- -A POEM. A poem she has composed herself. - -It's an awful poem, doggerel in the rollicking meter of Robert W. Service's -The Cremation of Sam McGee, but it is in fact, a poem. It's the Ballad -of the Electronic Frontier! A poem about the Hacker Crackdown and the -sheer unlikelihood of CFP. It's full of in-jokes. The score or so cops -in the audience, who are sitting together in a nervous claque, -are absolutely cracking-up. Gail's poem is the funniest goddamn thing -they've ever heard. The hackers and civil-libs, who had this woman figured -for Ilsa She-Wolf of the SS, are staring with their jaws hanging loosely. -Never in the wildest reaches of their imagination had they figured -Gail Thackeray was capable of such a totally off-the-wall move. -You can see them punching their mental CONTROL-RESET buttons. -Jesus! This woman's a hacker weirdo! She's JUST LIKE US! -God, this changes everything! - -Al Bayse, computer technician for the FBI, had been the only cop -at the CPSR Roundtable, dragged there with his arm bent by -Dorothy Denning. He was guarded and tightlipped at CPSR Roundtable; -a "lion thrown to the Christians." - -At CFP, backed by a claque of cops, Bayse suddenly waxes eloquent -and even droll, describing the FBI's "NCIC 2000", a gigantic digital catalog -of criminal records, as if he has suddenly become some weird hybrid -of George Orwell and George Gobel. Tentatively, he makes an arcane -joke about statistical analysis. At least a third of the crowd laughs aloud. - -"They didn't laugh at that at my last speech," Bayse observes. -He had been addressing cops--STRAIGHT cops, not computer people. -It had been a worthy meeting, useful one supposes, but nothing like THIS. -There has never been ANYTHING like this. Without any prodding, -without any preparation, people in the audience simply begin to ask questions. -Longhairs, freaky people, mathematicians. Bayse is answering, politely, -frankly, fully, like a man walking on air. The ballroom's atmosphere -crackles with surreality. A female lawyer behind me breaks into a sweat -and a hot waft of surprisingly potent and musky perfume flows off -her pulse-points. - -People are giddy with laughter. People are interested, -fascinated, their eyes so wide and dark that they seem eroticized. -Unlikely daisy-chains form in the halls, around the bar, on the escalators: -cops with hackers, civil rights with FBI, Secret Service with phone phreaks. - -Gail Thackeray is at her crispest in a white wool sweater with a -tiny Secret Service logo. "I found Phiber Optik at the payphones, -and when he saw my sweater, he turned into a PILLAR OF SALT!" she chortles. - -Phiber discusses his case at much length with his arresting officer, -Don Delaney of the New York State Police. After an hour's chat, -the two of them look ready to begin singing "Auld Lang Syne." -Phiber finally finds the courage to get his worst complaint off his chest. -It isn't so much the arrest. It was the CHARGE. Pirating service -off 900 numbers. I'm a PROGRAMMER, Phiber insists. This lame charge -is going to hurt my reputation. It would have been cool to be busted -for something happening, like Section 1030 computer intrusion. -Maybe some kind of crime that's scarcely been invented yet. -Not lousy phone fraud. Phooey. - -Delaney seems regretful. He had a mountain of possible criminal charges -against Phiber Optik. The kid's gonna plead guilty anyway. He's a -first timer, they always plead. Coulda charged the kid with most anything, -and gotten the same result in the end. Delaney seems genuinely sorry -not to have gratified Phiber in this harmless fashion. Too late now. -Phiber's pled already. All water under the bridge. Whaddya gonna do? - -Delaney's got a good grasp on the hacker mentality. -He held a press conference after he busted a bunch of -Masters of Deception kids. Some journo had asked him: -"Would you describe these people as GENIUSES?" -Delaney's deadpan answer, perfect: "No, I would describe -these people as DEFENDANTS." Delaney busts a kid for -hacking codes with repeated random dialling. Tells the -press that NYNEX can track this stuff in no time flat nowadays, -and a kid has to be STUPID to do something so easy to catch. -Dead on again: hackers don't mind being thought of as Genghis Khan -by the straights, but if there's anything that really gets 'em -where they live, it's being called DUMB. - -Won't be as much fun for Phiber next time around. -As a second offender he's gonna see prison. -Hackers break the law. They're not geniuses, either. -They're gonna be defendants. And yet, Delaney muses over -a drink in the hotel bar, he has found it impossible to treat -them as common criminals. Delaney knows criminals. These kids, -by comparison, are clueless--there is just no crook vibe off of them, -they don't smell right, they're just not BAD. - -Delaney has seen a lot of action. He did Vietnam. -He's been shot at, he has shot people. He's a homicide -cop from New York. He has the appearance of a man who -has not only seen the shit hit the fan but has seen it splattered -across whole city blocks and left to ferment for years. -This guy has been around. - -He listens to Steve Jackson tell his story. The dreamy -game strategist has been dealt a bad hand. He has played -it for all he is worth. Under his nerdish SF-fan exterior -is a core of iron. Friends of his say Steve Jackson believes -in the rules, believes in fair play. He will never compromise -his principles, never give up. "Steve," Delaney says to -Steve Jackson, "they had some balls, whoever busted you. -You're all right!" Jackson, stunned, falls silent and -actually blushes with pleasure. - -Neidorf has grown up a lot in the past year. The kid is -a quick study, you gotta give him that. Dressed by his mom, -the fashion manager for a national clothing chain, -Missouri college techie-frat Craig Neidorf out-dappers -everyone at this gig but the toniest East Coast lawyers. -The iron jaws of prison clanged shut without him and now -law school beckons for Neidorf. He looks like a larval Congressman. - -Not a "hacker," our Mr. Neidorf. He's not interested -in computer science. Why should he be? He's not -interested in writing C code the rest of his life, -and besides, he's seen where the chips fall. -To the world of computer science he and Phrack -were just a curiosity. But to the world of law. . . . -The kid has learned where the bodies are buried. -He carries his notebook of press clippings wherever he goes. - -Phiber Optik makes fun of Neidorf for a Midwestern geek, -for believing that "Acid Phreak" does acid and listens to acid rock. -Hell no. Acid's never done ACID! Acid's into ACID HOUSE MUSIC. -Jesus. The very idea of doing LSD. Our PARENTS did LSD, ya clown. - -Thackeray suddenly turns upon Craig Neidorf the full lighthouse -glare of her attention and begins a determined half-hour attempt -to WIN THE BOY OVER. The Joan of Arc of Computer Crime is -GIVING CAREER ADVICE TO KNIGHT LIGHTNING! "Your experience -would be very valuable--a real asset," she tells him with -unmistakeable sixty-thousand-watt sincerity. Neidorf is fascinated. -He listens with unfeigned attention. He's nodding and saying yes ma'am. -Yes, Craig, you too can forget all about money and enter the glamorous -and horribly underpaid world of PROSECUTING COMPUTER CRIME! -You can put your former friends in prison--ooops. . . . - -You cannot go on dueling at modem's length indefinitely. -You cannot beat one another senseless with rolled-up press-clippings. -Sooner or later you have to come directly to grips. -And yet the very act of assembling here has changed -the entire situation drastically. John Quarterman, -author of The Matrix, explains the Internet at his symposium. -It is the largest news network in the world, it is growing -by leaps and bounds, and yet you cannot measure Internet because -you cannot stop it in place. It cannot stop, because there -is no one anywhere in the world with the authority to stop Internet. -It changes, yes, it grows, it embeds itself across the post-industrial, -postmodern world and it generates community wherever it -touches, and it is doing this all by itself. - -Phiber is different. A very fin de siecle kid, Phiber Optik. -Barlow says he looks like an Edwardian dandy. He does rather. -Shaven neck, the sides of his skull cropped hip-hop close, -unruly tangle of black hair on top that looks pomaded, -he stays up till four a.m. and misses all the sessions, -then hangs out in payphone booths with his acoustic coupler -gutsily CRACKING SYSTEMS RIGHT IN THE MIDST OF THE HEAVIEST -LAW ENFORCEMENT DUDES IN THE U.S., or at least PRETENDING to. . . . -Unlike "Frank Drake." Drake, who wrote Dorothy Denning out -of nowhere, and asked for an interview for his cheapo -cyberpunk fanzine, and then started grilling her on her ethics. -She was squirmin', too. . . . Drake, scarecrow-tall with his -floppy blond mohawk, rotting tennis shoes and black leather jacket -lettered ILLUMINATI in red, gives off an unmistakeable air -of the bohemian literatus. Drake is the kind of guy -who reads British industrial design magazines and appreciates -William Gibson because the quality of the prose is so tasty. -Drake could never touch a phone or a keyboard again, -and he'd still have the nose-ring and the blurry photocopied -fanzines and the sampled industrial music. He's a radical punk -with a desktop-publishing rig and an Internet address. -Standing next to Drake, the diminutive Phiber looks like he's -been physically coagulated out of phone-lines. Born to phreak. - -Dorothy Denning approaches Phiber suddenly. The two of them -are about the same height and body-build. Denning's blue eyes -flash behind the round window-frames of her glasses. -"Why did you say I was `quaint?'" she asks Phiber, quaintly. - -It's a perfect description but Phiber is nonplussed. . . -"Well, I uh, you know. . . ." - -"I also think you're quaint, Dorothy," I say, novelist to the rescue, -the journo gift of gab. . . . She is neat and dapper and yet there's -an arcane quality to her, something like a Pilgrim Maiden behind -leaded glass; if she were six inches high Dorothy Denning would look -great inside a china cabinet. . .The Cryptographeress. . . -The Cryptographrix. . .whatever. . . . Weirdly, Peter Denning looks -just like his wife, you could pick this gentleman out of a thousand guys -as the soulmate of Dorothy Denning. Wearing tailored slacks, -a spotless fuzzy varsity sweater, and a neatly knotted academician's tie. . . . -This fineboned, exquisitely polite, utterly civilized and hyperintelligent -couple seem to have emerged from some cleaner and finer parallel universe, -where humanity exists to do the Brain Teasers column in Scientific American. -Why does this Nice Lady hang out with these unsavory characters? - -Because the time has come for it, that's why. -Because she's the best there is at what she does. - -Donn Parker is here, the Great Bald Eagle of Computer Crime. . . . -With his bald dome, great height, and enormous Lincoln-like hands, -the great visionary pioneer of the field plows through the lesser mortals -like an icebreaker. . . . His eyes are fixed on the future with the -rigidity of a bronze statue. . . . Eventually, he tells his audience, -all business crime will be computer crime, because businesses will do -everything through computers. "Computer crime" as a category will vanish. - -In the meantime, passing fads will flourish and fail and evaporate. . . . -Parker's commanding, resonant voice is sphinxlike, everything is viewed -from some eldritch valley of deep historical abstraction. . . . -Yes, they've come and they've gone, these passing flaps in the world -of digital computation. . . . The radio-frequency emanation scandal. . . -KGB and MI5 and CIA do it every day, it's easy, but nobody else ever has. . . . -The salami-slice fraud, mostly mythical. . . . "Crimoids," he calls them. . . . -Computer viruses are the current crimoid champ, a lot less dangerous than -most people let on, but the novelty is fading and there's a crimoid vacuum at -the moment, the press is visibly hungering for something more outrageous. . . . -The Great Man shares with us a few speculations on the coming crimoids. . . . -Desktop Forgery! Wow. . . . Computers stolen just for the sake of the -information within them--data-napping! Happened in Britain a while ago, -could be the coming thing. . . . Phantom nodes in the Internet! - -Parker handles his overhead projector sheets with an ecclesiastical air. . . . -He wears a grey double-breasted suit, a light blue shirt, and a -very quiet tie of understated maroon and blue paisley. . . . -Aphorisms emerge from him with slow, leaden emphasis. . . . -There is no such thing as an adequately secure computer -when one faces a sufficiently powerful adversary. . . . -Deterrence is the most socially useful aspect of security. . . . -People are the primary weakness in all information systems. . . . -The entire baseline of computer security must be shifted upward. . . . -Don't ever violate your security by publicly describing -your security measures. . . . - -People in the audience are beginning to squirm, and yet -there is something about the elemental purity of this guy's -philosophy that compels uneasy respect. . . . Parker sounds -like the only sane guy left in the lifeboat, sometimes. -The guy who can prove rigorously, from deep moral principles, -that Harvey there, the one with the broken leg and the checkered past, -is the one who has to be, err. . .that is, Mr. Harvey is best placed -to make the necessary sacrifice for the security and indeed -the very survival of the rest of this lifeboat's crew. . . . -Computer security, Parker informs us mournfully, is a -nasty topic, and we wish we didn't have to have it. . . . -The security expert, armed with method and logic, must think--imagine-- -everything that the adversary might do before the adversary might -actually do it. It is as if the criminal's dark brain were an -extensive subprogram within the shining cranium of Donn Parker. -He is a Holmes whose Moriarty does not quite yet exist -and so must be perfectly simulated. - -CFP is a stellar gathering, with the giddiness of a wedding. -It is a happy time, a happy ending, they know their world -is changing forever tonight, and they're proud to have been there -to see it happen, to talk, to think, to help. - -And yet as night falls, a certain elegiac quality manifests itself, -as the crowd gathers beneath the chandeliers with their wineglasses -and dessert plates. Something is ending here, gone forever, -and it takes a while to pinpoint it. - -It is the End of the Amateurs. - - - - - - - - - -End of the Project Gutenberg EBook of Hacker Crackdown, by Bruce Sterling - -*** END OF THIS PROJECT GUTENBERG EBOOK HACKER CRACKDOWN *** - -***** This file should be named 101.txt or 101.zip ***** -This and all associated files of various formats will be found in: - http://www.gutenberg.org/1/0/101/ - - - -Updated editions will replace the previous one--the old editions will be -renamed. - -Creating the works from public domain print editions means that no one -owns a United States copyright in these works, so the Foundation (and -you!) can copy and distribute it in the United States without permission -and without paying copyright royalties. Special rules, set forth in the -General Terms of Use part of this license, apply to copying and -distributing Project Gutenberg-tm electronic works to protect the -PROJECT GUTENBERG-tm concept and trademark. Project Gutenberg is a -registered trademark, and may not be used if you charge for the eBooks, -unless you receive specific permission. If you do not charge anything -for copies of this eBook, complying with the rules is very easy. You may -use this eBook for nearly any purpose such as creation of derivative -works, reports, performances and research. They may be modified and -printed and given away--you may do practically ANYTHING with public -domain eBooks. Redistribution is subject to the trademark license, -especially commercial redistribution. - - - -*** START: FULL LICENSE *** - -THE FULL PROJECT GUTENBERG LICENSE -PLEASE READ THIS BEFORE YOU DISTRIBUTE OR USE THIS WORK - -To protect the Project Gutenberg-tm mission of promoting the free -distribution of electronic works, by using or distributing this work -(or any other work associated in any way with the phrase "Project -Gutenberg"), you agree to comply with all the terms of the Full Project -Gutenberg-tm License (available with this file or online at -http://www.gutenberg.org/license). - - -Section 1. General Terms of Use and Redistributing Project Gutenberg-tm -electronic works - -1.A. By reading or using any part of this Project Gutenberg-tm -electronic work, you indicate that you have read, understand, agree to -and accept all the terms of this license and intellectual property -(trademark/copyright) agreement. If you do not agree to abide by all -the terms of this agreement, you must cease using and return or destroy -all copies of Project Gutenberg-tm electronic works in your possession. -If you paid a fee for obtaining a copy of or access to a Project -Gutenberg-tm electronic work and you do not agree to be bound by the -terms of this agreement, you may obtain a refund from the person or -entity to whom you paid the fee as set forth in paragraph 1.E.8. - -1.B. "Project Gutenberg" is a registered trademark. It may only be -used on or associated in any way with an electronic work by people who -agree to be bound by the terms of this agreement. There are a few -things that you can do with most Project Gutenberg-tm electronic works -even without complying with the full terms of this agreement. See -paragraph 1.C below. There are a lot of things you can do with Project -Gutenberg-tm electronic works if you follow the terms of this agreement -and help preserve free future access to Project Gutenberg-tm electronic -works. See paragraph 1.E below. - -1.C. The Project Gutenberg Literary Archive Foundation ("the Foundation" -or PGLAF), owns a compilation copyright in the collection of Project -Gutenberg-tm electronic works. Nearly all the individual works in the -collection are in the public domain in the United States. If an -individual work is in the public domain in the United States and you are -located in the United States, we do not claim a right to prevent you from -copying, distributing, performing, displaying or creating derivative -works based on the work as long as all references to Project Gutenberg -are removed. Of course, we hope that you will support the Project -Gutenberg-tm mission of promoting free access to electronic works by -freely sharing Project Gutenberg-tm works in compliance with the terms of -this agreement for keeping the Project Gutenberg-tm name associated with -the work. You can easily comply with the terms of this agreement by -keeping this work in the same format with its attached full Project -Gutenberg-tm License when you share it without charge with others. -This particular work is one of the few copyrighted individual works -included with the permission of the copyright holder. Information on -the copyright owner for this particular work and the terms of use -imposed by the copyright holder on this work are set forth at the -beginning of this work. - -1.D. The copyright laws of the place where you are located also govern -what you can do with this work. Copyright laws in most countries are in -a constant state of change. If you are outside the United States, check -the laws of your country in addition to the terms of this agreement -before downloading, copying, displaying, performing, distributing or -creating derivative works based on this work or any other Project -Gutenberg-tm work. The Foundation makes no representations concerning -the copyright status of any work in any country outside the United -States. - -1.E. Unless you have removed all references to Project Gutenberg: - -1.E.1. The following sentence, with active links to, or other immediate -access to, the full Project Gutenberg-tm License must appear prominently -whenever any copy of a Project Gutenberg-tm work (any work on which the -phrase "Project Gutenberg" appears, or with which the phrase "Project -Gutenberg" is associated) is accessed, displayed, performed, viewed, -copied or distributed: - -This eBook is for the use of anyone anywhere at no cost and with -almost no restrictions whatsoever. You may copy it, give it away or -re-use it under the terms of the Project Gutenberg License included -with this eBook or online at www.gutenberg.org - -1.E.2. If an individual Project Gutenberg-tm electronic work is derived -from the public domain (does not contain a notice indicating that it is -posted with permission of the copyright holder), the work can be copied -and distributed to anyone in the United States without paying any fees -or charges. If you are redistributing or providing access to a work -with the phrase "Project Gutenberg" associated with or appearing on the -work, you must comply either with the requirements of paragraphs 1.E.1 -through 1.E.7 or obtain permission for the use of the work and the -Project Gutenberg-tm trademark as set forth in paragraphs 1.E.8 or -1.E.9. - -1.E.3. If an individual Project Gutenberg-tm electronic work is posted -with the permission of the copyright holder, your use and distribution -must comply with both paragraphs 1.E.1 through 1.E.7 and any additional -terms imposed by the copyright holder. Additional terms will be linked -to the Project Gutenberg-tm License for all works posted with the -permission of the copyright holder found at the beginning of this work. - -1.E.4. Do not unlink or detach or remove the full Project Gutenberg-tm -License terms from this work, or any files containing a part of this -work or any other work associated with Project Gutenberg-tm. - -1.E.5. Do not copy, display, perform, distribute or redistribute this -electronic work, or any part of this electronic work, without -prominently displaying the sentence set forth in paragraph 1.E.1 with -active links or immediate access to the full terms of the Project -Gutenberg-tm License. - -1.E.6. You may convert to and distribute this work in any binary, -compressed, marked up, nonproprietary or proprietary form, including any -word processing or hypertext form. However, if you provide access to or -distribute copies of a Project Gutenberg-tm work in a format other than -"Plain Vanilla ASCII" or other format used in the official version -posted on the official Project Gutenberg-tm web site (www.gutenberg.org), -you must, at no additional cost, fee or expense to the user, provide a -copy, a means of exporting a copy, or a means of obtaining a copy upon -request, of the work in its original "Plain Vanilla ASCII" or other -form. Any alternate format must include the full Project Gutenberg-tm -License as specified in paragraph 1.E.1. - -1.E.7. Do not charge a fee for access to, viewing, displaying, -performing, copying or distributing any Project Gutenberg-tm works -unless you comply with paragraph 1.E.8 or 1.E.9. - -1.E.8. You may charge a reasonable fee for copies of or providing -access to or distributing Project Gutenberg-tm electronic works provided -that - -- You pay a royalty fee of 20% of the gross profits you derive from - the use of Project Gutenberg-tm works calculated using the method - you already use to calculate your applicable taxes. The fee is - owed to the owner of the Project Gutenberg-tm trademark, but he - has agreed to donate royalties under this paragraph to the - Project Gutenberg Literary Archive Foundation. Royalty payments - must be paid within 60 days following each date on which you - prepare (or are legally required to prepare) your periodic tax - returns. Royalty payments should be clearly marked as such and - sent to the Project Gutenberg Literary Archive Foundation at the - address specified in Section 4, "Information about donations to - the Project Gutenberg Literary Archive Foundation." - -- You provide a full refund of any money paid by a user who notifies - you in writing (or by e-mail) within 30 days of receipt that s/he - does not agree to the terms of the full Project Gutenberg-tm - License. You must require such a user to return or - destroy all copies of the works possessed in a physical medium - and discontinue all use of and all access to other copies of - Project Gutenberg-tm works. - -- You provide, in accordance with paragraph 1.F.3, a full refund of any - money paid for a work or a replacement copy, if a defect in the - electronic work is discovered and reported to you within 90 days - of receipt of the work. - -- You comply with all other terms of this agreement for free - distribution of Project Gutenberg-tm works. - -1.E.9. If you wish to charge a fee or distribute a Project Gutenberg-tm -electronic work or group of works on different terms than are set -forth in this agreement, you must obtain permission in writing from -both the Project Gutenberg Literary Archive Foundation and Michael -Hart, the owner of the Project Gutenberg-tm trademark. Contact the -Foundation as set forth in Section 3 below. - -1.F. - -1.F.1. Project Gutenberg volunteers and employees expend considerable -effort to identify, do copyright research on, transcribe and proofread -public domain works in creating the Project Gutenberg-tm -collection. Despite these efforts, Project Gutenberg-tm electronic -works, and the medium on which they may be stored, may contain -"Defects," such as, but not limited to, incomplete, inaccurate or -corrupt data, transcription errors, a copyright or other intellectual -property infringement, a defective or damaged disk or other medium, a -computer virus, or computer codes that damage or cannot be read by -your equipment. - -1.F.2. LIMITED WARRANTY, DISCLAIMER OF DAMAGES - Except for the "Right -of Replacement or Refund" described in paragraph 1.F.3, the Project -Gutenberg Literary Archive Foundation, the owner of the Project -Gutenberg-tm trademark, and any other party distributing a Project -Gutenberg-tm electronic work under this agreement, disclaim all -liability to you for damages, costs and expenses, including legal -fees. YOU AGREE THAT YOU HAVE NO REMEDIES FOR NEGLIGENCE, STRICT -LIABILITY, BREACH OF WARRANTY OR BREACH OF CONTRACT EXCEPT THOSE -PROVIDED IN PARAGRAPH 1.F.3. YOU AGREE THAT THE FOUNDATION, THE -TRADEMARK OWNER, AND ANY DISTRIBUTOR UNDER THIS AGREEMENT WILL NOT BE -LIABLE TO YOU FOR ACTUAL, DIRECT, INDIRECT, CONSEQUENTIAL, PUNITIVE OR -INCIDENTAL DAMAGES EVEN IF YOU GIVE NOTICE OF THE POSSIBILITY OF SUCH -DAMAGE. - -1.F.3. LIMITED RIGHT OF REPLACEMENT OR REFUND - If you discover a -defect in this electronic work within 90 days of receiving it, you can -receive a refund of the money (if any) you paid for it by sending a -written explanation to the person you received the work from. If you -received the work on a physical medium, you must return the medium with -your written explanation. The person or entity that provided you with -the defective work may elect to provide a replacement copy in lieu of a -refund. If you received the work electronically, the person or entity -providing it to you may choose to give you a second opportunity to -receive the work electronically in lieu of a refund. If the second copy -is also defective, you may demand a refund in writing without further -opportunities to fix the problem. - -1.F.4. Except for the limited right of replacement or refund set forth -in paragraph 1.F.3, this work is provided to you 'AS-IS,' WITH NO OTHER -WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -WARRANTIES OF MERCHANTIBILITY OR FITNESS FOR ANY PURPOSE. - -1.F.5. Some states do not allow disclaimers of certain implied -warranties or the exclusion or limitation of certain types of damages. -If any disclaimer or limitation set forth in this agreement violates the -law of the state applicable to this agreement, the agreement shall be -interpreted to make the maximum disclaimer or limitation permitted by -the applicable state law. The invalidity or unenforceability of any -provision of this agreement shall not void the remaining provisions. - -1.F.6. INDEMNITY - You agree to indemnify and hold the Foundation, the -trademark owner, any agent or employee of the Foundation, anyone -providing copies of Project Gutenberg-tm electronic works in accordance -with this agreement, and any volunteers associated with the production, -promotion and distribution of Project Gutenberg-tm electronic works, -harmless from all liability, costs and expenses, including legal fees, -that arise directly or indirectly from any of the following which you do -or cause to occur: (a) distribution of this or any Project Gutenberg-tm -work, (b) alteration, modification, or additions or deletions to any -Project Gutenberg-tm work, and (c) any Defect you cause. - - -Section 2. Information about the Mission of Project Gutenberg-tm - -Project Gutenberg-tm is synonymous with the free distribution of -electronic works in formats readable by the widest variety of computers -including obsolete, old, middle-aged and new computers. It exists -because of the efforts of hundreds of volunteers and donations from -people in all walks of life. - -Volunteers and financial support to provide volunteers with the -assistance they need are critical to reaching Project Gutenberg-tm's -goals and ensuring that the Project Gutenberg-tm collection will -remain freely available for generations to come. In 2001, the Project -Gutenberg Literary Archive Foundation was created to provide a secure -and permanent future for Project Gutenberg-tm and future generations. -To learn more about the Project Gutenberg Literary Archive Foundation -and how your efforts and donations can help, see Sections 3 and 4 -and the Foundation web page at http://www.pglaf.org. - - -Section 3. Information about the Project Gutenberg Literary Archive -Foundation - -The Project Gutenberg Literary Archive Foundation is a non profit -501(c)(3) educational corporation organized under the laws of the -state of Mississippi and granted tax exempt status by the Internal -Revenue Service. The Foundation's EIN or federal tax identification -number is 64-6221541. Its 501(c)(3) letter is posted at -http://pglaf.org/fundraising. Contributions to the Project Gutenberg -Literary Archive Foundation are tax deductible to the full extent -permitted by U.S. federal laws and your state's laws. - -The Foundation's principal office is located at 4557 Melan Dr. S. -Fairbanks, AK, 99712., but its volunteers and employees are scattered -throughout numerous locations. Its business office is located at -809 North 1500 West, Salt Lake City, UT 84116, (801) 596-1887, email -business@pglaf.org. Email contact links and up to date contact -information can be found at the Foundation's web site and official -page at http://pglaf.org - -For additional contact information: - Dr. Gregory B. Newby - Chief Executive and Director - gbnewby@pglaf.org - -Section 4. Information about Donations to the Project Gutenberg -Literary Archive Foundation - -Project Gutenberg-tm depends upon and cannot survive without wide -spread public support and donations to carry out its mission of -increasing the number of public domain and licensed works that can be -freely distributed in machine readable form accessible by the widest -array of equipment including outdated equipment. Many small donations -($1 to $5,000) are particularly important to maintaining tax exempt -status with the IRS. - -The Foundation is committed to complying with the laws regulating -charities and charitable donations in all 50 states of the United -States. Compliance requirements are not uniform and it takes a -considerable effort, much paperwork and many fees to meet and keep up -with these requirements. We do not solicit donations in locations -where we have not received written confirmation of compliance. To -SEND DONATIONS or determine the status of compliance for any -particular state visit http://pglaf.org - -While we cannot and do not solicit contributions from states where we -have not met the solicitation requirements, we know of no prohibition -against accepting unsolicited donations from donors in such states who -approach us with offers to donate. - -International donations are gratefully accepted, but we cannot make -any statements concerning tax treatment of donations received from -outside the United States. U.S. laws alone swamp our small staff. - -Please check the Project Gutenberg Web pages for current donation -methods and addresses. Donations are accepted in a number of other -ways including checks, online payments and credit card donations. -To donate, please visit: http://pglaf.org/donate - - -Section 5. General Information About Project Gutenberg-tm electronic -works. - -Professor Michael S. Hart is the originator of the Project Gutenberg-tm -concept of a library of electronic works that could be freely shared -with anyone. For thirty years, he produced and distributed Project -Gutenberg-tm eBooks with only a loose network of volunteer support. - -Project Gutenberg-tm eBooks are often created from several printed -editions, all of which are confirmed as Public Domain in the U.S. -unless a copyright notice is included. Thus, we do not necessarily -keep eBooks in compliance with any particular paper edition. - -Each eBook is in a subdirectory of the same number as the eBook's -eBook number, often in several formats including plain vanilla ASCII, -compressed (zipped), HTML and others. - -Corrected EDITIONS of our eBooks replace the old file and take over -the old filename and etext number. The replaced older file is renamed. -VERSIONS based on separate sources are treated as new eBooks receiving -new filenames and etext numbers. - -Most people start at our Web site which has the main PG search facility: - -http://www.gutenberg.org - -This Web site includes information about Project Gutenberg-tm, -including how to make donations to the Project Gutenberg Literary -Archive Foundation, how to help produce our new eBooks, and how to -subscribe to our email newsletter to hear about new eBooks. - -EBooks posted prior to November 2003, with eBook numbers BELOW #10000, -are filed in directories based on their release date. If you want to -download any of these eBooks directly, rather than using the regular -search system you may utilize the following addresses and just -download by the etext year. - -http://www.ibiblio.org/gutenberg/etext06 - - (Or /etext 05, 04, 03, 02, 01, 00, 99, - 98, 97, 96, 95, 94, 93, 92, 92, 91 or 90) - -EBooks posted since November 2003, with etext numbers OVER #10000, are -filed in a different way. The year of a release date is no longer part -of the directory path. The path is based on the etext number (which is -identical to the filename). The path to the file is made up of single -digits corresponding to all but the last digit in the filename. For -example an eBook of filename 10234 would be found at: - -http://www.gutenberg.org/1/0/2/3/10234 - -or filename 24689 would be found at: -http://www.gutenberg.org/2/4/6/8/24689 - -An alternative method of locating eBooks: -http://www.gutenberg.org/GUTINDEX.ALL - -*** END: FULL LICENSE *** diff --git a/testing/tests/couch/common.py b/testing/tests/couch/common.py new file mode 100644 index 00000000..b08e1fa3 --- /dev/null +++ b/testing/tests/couch/common.py @@ -0,0 +1,81 @@ +from uuid import uuid4 +from urlparse import urljoin +from couchdb.client import Server + +from leap.soledad.common import couch +from leap.soledad.common.document import ServerDocument + +from test_soledad import u1db_tests as tests + + +simple_doc = tests.simple_doc +nested_doc = tests.nested_doc + + +def make_couch_database_for_test(test, replica_uid): + port = str(test.couch_port) + dbname = ('test-%s' % uuid4().hex) + db = couch.CouchDatabase.open_database( + urljoin('http://localhost:' + port, dbname), + create=True, + replica_uid=replica_uid or 'test', + ensure_ddocs=True) + test.addCleanup(test.delete_db, dbname) + return db + + +def copy_couch_database_for_test(test, db): + port = str(test.couch_port) + couch_url = 'http://localhost:' + port + new_dbname = db._dbname + '_copy' + new_db = couch.CouchDatabase.open_database( + urljoin(couch_url, new_dbname), + create=True, + replica_uid=db._replica_uid or 'test') + # copy all docs + session = couch.Session() + old_couch_db = Server(couch_url, session=session)[db._dbname] + new_couch_db = Server(couch_url, session=session)[new_dbname] + for doc_id in old_couch_db: + doc = old_couch_db.get(doc_id) + # bypass u1db_config document + if doc_id == 'u1db_config': + pass + # copy design docs + elif doc_id.startswith('_design'): + del doc['_rev'] + new_couch_db.save(doc) + # copy u1db docs + elif 'u1db_rev' in doc: + new_doc = { + '_id': doc['_id'], + 'u1db_transactions': doc['u1db_transactions'], + 'u1db_rev': doc['u1db_rev'] + } + attachments = [] + if ('u1db_conflicts' in doc): + new_doc['u1db_conflicts'] = doc['u1db_conflicts'] + for c_rev in doc['u1db_conflicts']: + attachments.append('u1db_conflict_%s' % c_rev) + new_couch_db.save(new_doc) + # save conflict data + attachments.append('u1db_content') + for att_name in attachments: + att = old_couch_db.get_attachment(doc_id, att_name) + if (att is not None): + new_couch_db.put_attachment(new_doc, att, + filename=att_name) + # cleanup connections to prevent file descriptor leaking + return new_db + + +def make_document_for_test(test, doc_id, rev, content, has_conflicts=False): + return ServerDocument( + doc_id, rev, content, has_conflicts=has_conflicts) + + +COUCH_SCENARIOS = [ + ('couch', {'make_database_for_test': make_couch_database_for_test, + 'copy_database_for_test': copy_couch_database_for_test, + 'make_document_for_test': make_document_for_test, }), +] diff --git a/testing/tests/couch/test_atomicity.py b/testing/tests/couch/test_atomicity.py new file mode 100644 index 00000000..aec9c6cf --- /dev/null +++ b/testing/tests/couch/test_atomicity.py @@ -0,0 +1,371 @@ +# -*- coding: utf-8 -*- +# test_couch_operations_atomicity.py +# Copyright (C) 2013, 2014 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test atomicity of couch operations. +""" +import os +import tempfile +import threading + +from urlparse import urljoin +from twisted.internet import defer +from uuid import uuid4 + +from leap.soledad.client import Soledad +from leap.soledad.common.couch.state import CouchServerState +from leap.soledad.common.couch import CouchDatabase + +from test_soledad.util import ( + make_token_soledad_app, + make_soledad_document_for_test, + soledad_sync_target, +) +from test_soledad.util import CouchDBTestCase +from test_soledad.u1db_tests import TestCaseWithServer + + +REPEAT_TIMES = 20 + + +class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): + + @staticmethod + def make_app_after_state(state): + return make_token_soledad_app(state) + + make_document_for_test = make_soledad_document_for_test + + sync_target = soledad_sync_target + + def _soledad_instance(self, user=None, passphrase=u'123', + prefix='', + secrets_path='secrets.json', + local_db_path='soledad.u1db', server_url='', + cert_file=None, auth_token=None): + """ + Instantiate Soledad. + """ + user = user or self.user + + # this callback ensures we save a document which is sent to the shared + # db. + def _put_doc_side_effect(doc): + self._doc_put = doc + + soledad = Soledad( + user, + passphrase, + secrets_path=os.path.join(self.tempdir, prefix, secrets_path), + local_db_path=os.path.join( + self.tempdir, prefix, local_db_path), + server_url=server_url, + cert_file=cert_file, + auth_token=auth_token, + shared_db=self.get_default_shared_mock(_put_doc_side_effect)) + self.addCleanup(soledad.close) + return soledad + + def make_app(self): + self.request_state = CouchServerState(self.couch_url) + return self.make_app_after_state(self.request_state) + + def setUp(self): + TestCaseWithServer.setUp(self) + CouchDBTestCase.setUp(self) + self.user = ('user-%s' % uuid4().hex) + self.db = CouchDatabase.open_database( + urljoin(self.couch_url, 'user-' + self.user), + create=True, + replica_uid='replica', + ensure_ddocs=True) + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + self.startTwistedServer() + + def tearDown(self): + self.db.delete_database() + self.db.close() + CouchDBTestCase.tearDown(self) + TestCaseWithServer.tearDown(self) + + # + # Sequential tests + # + + def test_correct_transaction_log_after_sequential_puts(self): + """ + Assert that the transaction_log increases accordingly with sequential + puts. + """ + doc = self.db.create_doc({'ops': 0}) + docs = [doc.doc_id] + for i in range(0, REPEAT_TIMES): + self.assertEqual( + i + 1, len(self.db._get_transaction_log())) + doc.content['ops'] += 1 + self.db.put_doc(doc) + docs.append(doc.doc_id) + + # assert length of transaction_log + transaction_log = self.db._get_transaction_log() + self.assertEqual( + REPEAT_TIMES + 1, len(transaction_log)) + + # assert that all entries in the log belong to the same doc + self.assertEqual(REPEAT_TIMES + 1, len(docs)) + for doc_id in docs: + self.assertEqual( + REPEAT_TIMES + 1, + len(filter(lambda t: t[0] == doc_id, transaction_log))) + + def test_correct_transaction_log_after_sequential_deletes(self): + """ + Assert that the transaction_log increases accordingly with sequential + puts and deletes. + """ + docs = [] + for i in range(0, REPEAT_TIMES): + doc = self.db.create_doc({'ops': 0}) + self.assertEqual( + 2 * i + 1, len(self.db._get_transaction_log())) + docs.append(doc.doc_id) + self.db.delete_doc(doc) + self.assertEqual( + 2 * i + 2, len(self.db._get_transaction_log())) + + # assert length of transaction_log + transaction_log = self.db._get_transaction_log() + self.assertEqual( + 2 * REPEAT_TIMES, len(transaction_log)) + + # assert that each doc appears twice in the transaction_log + self.assertEqual(REPEAT_TIMES, len(docs)) + for doc_id in docs: + self.assertEqual( + 2, + len(filter(lambda t: t[0] == doc_id, transaction_log))) + + @defer.inlineCallbacks + def test_correct_sync_log_after_sequential_syncs(self): + """ + Assert that the sync_log increases accordingly with sequential syncs. + """ + sol = self._soledad_instance( + auth_token='auth-token', + server_url=self.getURL()) + source_replica_uid = sol._dbpool.replica_uid + + def _create_docs(): + deferreds = [] + for i in xrange(0, REPEAT_TIMES): + deferreds.append(sol.create_doc({})) + return defer.gatherResults(deferreds) + + def _assert_transaction_and_sync_logs(results, sync_idx): + # assert sizes of transaction and sync logs + self.assertEqual( + sync_idx * REPEAT_TIMES, + len(self.db._get_transaction_log())) + gen, _ = self.db._get_replica_gen_and_trans_id(source_replica_uid) + self.assertEqual(sync_idx * REPEAT_TIMES, gen) + + def _assert_sync(results, sync_idx): + gen, docs = results + self.assertEqual((sync_idx + 1) * REPEAT_TIMES, gen) + self.assertEqual((sync_idx + 1) * REPEAT_TIMES, len(docs)) + # assert sizes of transaction and sync logs + self.assertEqual((sync_idx + 1) * REPEAT_TIMES, + len(self.db._get_transaction_log())) + target_known_gen, target_known_trans_id = \ + self.db._get_replica_gen_and_trans_id(source_replica_uid) + # assert it has the correct gen and trans_id + conn_key = sol._dbpool._u1dbconnections.keys().pop() + conn = sol._dbpool._u1dbconnections[conn_key] + sol_gen, sol_trans_id = conn._get_generation_info() + self.assertEqual(sol_gen, target_known_gen) + self.assertEqual(sol_trans_id, target_known_trans_id) + + # sync first time and assert success + results = yield _create_docs() + _assert_transaction_and_sync_logs(results, 0) + yield sol.sync() + results = yield sol.get_all_docs() + _assert_sync(results, 0) + + # create more docs, sync second time and assert success + results = yield _create_docs() + _assert_transaction_and_sync_logs(results, 1) + yield sol.sync() + results = yield sol.get_all_docs() + _assert_sync(results, 1) + + # + # Concurrency tests + # + + class _WorkerThread(threading.Thread): + + def __init__(self, params, run_method): + threading.Thread.__init__(self) + self._params = params + self._run_method = run_method + + def run(self): + self._run_method(self) + + def test_correct_transaction_log_after_concurrent_puts(self): + """ + Assert that the transaction_log increases accordingly with concurrent + puts. + """ + pool = threading.BoundedSemaphore(value=1) + threads = [] + docs = [] + + def _run_method(self): + doc = self._params['db'].create_doc({}) + pool.acquire() + self._params['docs'].append(doc.doc_id) + pool.release() + + for i in range(0, REPEAT_TIMES): + thread = self._WorkerThread( + {'docs': docs, 'db': self.db}, + _run_method) + thread.start() + threads.append(thread) + + for thread in threads: + thread.join() + + # assert length of transaction_log + transaction_log = self.db._get_transaction_log() + self.assertEqual( + REPEAT_TIMES, len(transaction_log)) + + # assert all documents are in the log + self.assertEqual(REPEAT_TIMES, len(docs)) + for doc_id in docs: + self.assertEqual( + 1, + len(filter(lambda t: t[0] == doc_id, transaction_log))) + + def test_correct_transaction_log_after_concurrent_deletes(self): + """ + Assert that the transaction_log increases accordingly with concurrent + puts and deletes. + """ + threads = [] + docs = [] + pool = threading.BoundedSemaphore(value=1) + + # create/delete method that will be run concurrently + def _run_method(self): + doc = self._params['db'].create_doc({}) + pool.acquire() + docs.append(doc.doc_id) + pool.release() + self._params['db'].delete_doc(doc) + + # launch concurrent threads + for i in range(0, REPEAT_TIMES): + thread = self._WorkerThread({'db': self.db}, _run_method) + thread.start() + threads.append(thread) + + # wait for threads to finish + for thread in threads: + thread.join() + + # assert transaction log + transaction_log = self.db._get_transaction_log() + self.assertEqual( + 2 * REPEAT_TIMES, len(transaction_log)) + # assert that each doc appears twice in the transaction_log + self.assertEqual(REPEAT_TIMES, len(docs)) + for doc_id in docs: + self.assertEqual( + 2, + len(filter(lambda t: t[0] == doc_id, transaction_log))) + + def test_correct_sync_log_after_concurrent_puts_and_sync(self): + """ + Assert that the sync_log is correct after concurrent syncs. + """ + docs = [] + + sol = self._soledad_instance( + auth_token='auth-token', + server_url=self.getURL()) + + def _save_doc_ids(results): + for doc in results: + docs.append(doc.doc_id) + + # create documents in parallel + deferreds = [] + for i in range(0, REPEAT_TIMES): + d = sol.create_doc({}) + deferreds.append(d) + + # wait for documents creation and sync + d = defer.gatherResults(deferreds) + d.addCallback(_save_doc_ids) + d.addCallback(lambda _: sol.sync()) + + def _assert_logs(results): + transaction_log = self.db._get_transaction_log() + self.assertEqual(REPEAT_TIMES, len(transaction_log)) + # assert all documents are in the remote log + self.assertEqual(REPEAT_TIMES, len(docs)) + for doc_id in docs: + self.assertEqual( + 1, + len(filter(lambda t: t[0] == doc_id, transaction_log))) + + d.addCallback(_assert_logs) + d.addCallback(lambda _: sol.close()) + + return d + + @defer.inlineCallbacks + def test_concurrent_syncs_do_not_fail(self): + """ + Assert that concurrent attempts to sync end up being executed + sequentially and do not fail. + """ + docs = [] + + sol = self._soledad_instance( + auth_token='auth-token', + server_url=self.getURL()) + + deferreds = [] + for i in xrange(0, REPEAT_TIMES): + d = sol.create_doc({}) + d.addCallback(lambda doc: docs.append(doc.doc_id)) + d.addCallback(lambda _: sol.sync()) + deferreds.append(d) + yield defer.gatherResults(deferreds, consumeErrors=True) + + transaction_log = self.db._get_transaction_log() + self.assertEqual(REPEAT_TIMES, len(transaction_log)) + # assert all documents are in the remote log + self.assertEqual(REPEAT_TIMES, len(docs)) + for doc_id in docs: + self.assertEqual( + 1, + len(filter(lambda t: t[0] == doc_id, transaction_log))) diff --git a/testing/tests/couch/test_backend.py b/testing/tests/couch/test_backend.py new file mode 100644 index 00000000..f178e8a5 --- /dev/null +++ b/testing/tests/couch/test_backend.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +# test_couch.py +# Copyright (C) 2013-2016 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test ObjectStore and Couch backend bits. +""" + +from uuid import uuid4 +from urlparse import urljoin +from testscenarios import TestWithScenarios +from twisted.trial import unittest + +from leap.soledad.common import couch + +from test_soledad.util import CouchDBTestCase +from test_soledad.u1db_tests import test_backends + +from common import COUCH_SCENARIOS + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_common_backend`. +# ----------------------------------------------------------------------------- + +class TestCouchBackendImpl(CouchDBTestCase): + + def test__allocate_doc_id(self): + db = couch.CouchDatabase.open_database( + urljoin( + 'http://localhost:' + str(self.couch_port), + ('test-%s' % uuid4().hex) + ), + create=True, + ensure_ddocs=True) + doc_id1 = db._allocate_doc_id() + self.assertTrue(doc_id1.startswith('D-')) + self.assertEqual(34, len(doc_id1)) + int(doc_id1[len('D-'):], 16) + self.assertNotEqual(doc_id1, db._allocate_doc_id()) + self.delete_db(db._dbname) + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_backends`. +# ----------------------------------------------------------------------------- + +class CouchTests( + TestWithScenarios, test_backends.AllDatabaseTests, CouchDBTestCase): + + scenarios = COUCH_SCENARIOS + + +class CouchBackendTests( + TestWithScenarios, + test_backends.LocalDatabaseTests, + CouchDBTestCase): + + scenarios = COUCH_SCENARIOS + + +class CouchValidateGenNTransIdTests( + TestWithScenarios, + test_backends.LocalDatabaseValidateGenNTransIdTests, + CouchDBTestCase): + + scenarios = COUCH_SCENARIOS + + +class CouchValidateSourceGenTests( + TestWithScenarios, + test_backends.LocalDatabaseValidateSourceGenTests, + CouchDBTestCase): + + scenarios = COUCH_SCENARIOS + + +class CouchWithConflictsTests( + TestWithScenarios, + test_backends.LocalDatabaseWithConflictsTests, + CouchDBTestCase): + + scenarios = COUCH_SCENARIOS + + +# Notice: the CouchDB backend does not have indexing capabilities, so we do +# not test indexing now. + +# class CouchIndexTests(test_backends.DatabaseIndexTests, CouchDBTestCase): +# +# scenarios = COUCH_SCENARIOS +# +# def tearDown(self): +# self.db.delete_database() +# test_backends.DatabaseIndexTests.tearDown(self) + + +class DatabaseNameValidationTest(unittest.TestCase): + + def test_database_name_validation(self): + inject = couch.state.is_db_name_valid("user-deadbeef | cat /secret") + self.assertFalse(inject) + self.assertTrue(couch.state.is_db_name_valid("user-cafe1337")) diff --git a/testing/tests/couch/test_command.py b/testing/tests/couch/test_command.py new file mode 100644 index 00000000..f61e118d --- /dev/null +++ b/testing/tests/couch/test_command.py @@ -0,0 +1,28 @@ +from twisted.trial import unittest + +from leap.soledad.common import couch +from leap.soledad.common.l2db import errors as u1db_errors + +from mock import Mock + + +class CommandBasedDBCreationTest(unittest.TestCase): + + def test_ensure_db_using_custom_command(self): + state = couch.state.CouchServerState("url", create_cmd="echo") + mock_db = Mock() + mock_db.replica_uid = 'replica_uid' + state.open_database = Mock(return_value=mock_db) + db, replica_uid = state.ensure_database("user-1337") # works + self.assertEquals(mock_db, db) + self.assertEquals(mock_db.replica_uid, replica_uid) + + def test_raises_unauthorized_on_failure(self): + state = couch.state.CouchServerState("url", create_cmd="inexistent") + self.assertRaises(u1db_errors.Unauthorized, + state.ensure_database, "user-1337") + + def test_raises_unauthorized_by_default(self): + state = couch.state.CouchServerState("url") + self.assertRaises(u1db_errors.Unauthorized, + state.ensure_database, "user-1337") diff --git a/testing/tests/couch/test_couch.py b/testing/tests/couch/test_couch.py deleted file mode 100644 index 94c6ca92..00000000 --- a/testing/tests/couch/test_couch.py +++ /dev/null @@ -1,1442 +0,0 @@ -# -*- coding: utf-8 -*- -# test_couch.py -# Copyright (C) 2013-2016 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Test ObjectStore and Couch backend bits. -""" -import json - -from uuid import uuid4 -from urlparse import urljoin - -from couchdb.client import Server - -from testscenarios import TestWithScenarios -from twisted.trial import unittest -from mock import Mock - -from leap.soledad.common.l2db import errors as u1db_errors -from leap.soledad.common.l2db import SyncTarget -from leap.soledad.common.l2db import vectorclock - -from leap.soledad.common import couch -from leap.soledad.common.document import ServerDocument -from leap.soledad.common.couch import errors - -from test_soledad import u1db_tests as tests -from test_soledad.util import CouchDBTestCase -from test_soledad.util import make_local_db_and_target -from test_soledad.util import sync_via_synchronizer - -from test_soledad.u1db_tests import test_backends -from test_soledad.u1db_tests import DatabaseBaseTests - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_common_backend`. -# ----------------------------------------------------------------------------- - -class TestCouchBackendImpl(CouchDBTestCase): - - def test__allocate_doc_id(self): - db = couch.CouchDatabase.open_database( - urljoin( - 'http://localhost:' + str(self.couch_port), - ('test-%s' % uuid4().hex) - ), - create=True, - ensure_ddocs=True) - doc_id1 = db._allocate_doc_id() - self.assertTrue(doc_id1.startswith('D-')) - self.assertEqual(34, len(doc_id1)) - int(doc_id1[len('D-'):], 16) - self.assertNotEqual(doc_id1, db._allocate_doc_id()) - self.delete_db(db._dbname) - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_backends`. -# ----------------------------------------------------------------------------- - -def make_couch_database_for_test(test, replica_uid): - port = str(test.couch_port) - dbname = ('test-%s' % uuid4().hex) - db = couch.CouchDatabase.open_database( - urljoin('http://localhost:' + port, dbname), - create=True, - replica_uid=replica_uid or 'test', - ensure_ddocs=True) - test.addCleanup(test.delete_db, dbname) - return db - - -def copy_couch_database_for_test(test, db): - port = str(test.couch_port) - couch_url = 'http://localhost:' + port - new_dbname = db._dbname + '_copy' - new_db = couch.CouchDatabase.open_database( - urljoin(couch_url, new_dbname), - create=True, - replica_uid=db._replica_uid or 'test') - # copy all docs - session = couch.Session() - old_couch_db = Server(couch_url, session=session)[db._dbname] - new_couch_db = Server(couch_url, session=session)[new_dbname] - for doc_id in old_couch_db: - doc = old_couch_db.get(doc_id) - # bypass u1db_config document - if doc_id == 'u1db_config': - pass - # copy design docs - elif doc_id.startswith('_design'): - del doc['_rev'] - new_couch_db.save(doc) - # copy u1db docs - elif 'u1db_rev' in doc: - new_doc = { - '_id': doc['_id'], - 'u1db_transactions': doc['u1db_transactions'], - 'u1db_rev': doc['u1db_rev'] - } - attachments = [] - if ('u1db_conflicts' in doc): - new_doc['u1db_conflicts'] = doc['u1db_conflicts'] - for c_rev in doc['u1db_conflicts']: - attachments.append('u1db_conflict_%s' % c_rev) - new_couch_db.save(new_doc) - # save conflict data - attachments.append('u1db_content') - for att_name in attachments: - att = old_couch_db.get_attachment(doc_id, att_name) - if (att is not None): - new_couch_db.put_attachment(new_doc, att, - filename=att_name) - # cleanup connections to prevent file descriptor leaking - return new_db - - -def make_document_for_test(test, doc_id, rev, content, has_conflicts=False): - return ServerDocument( - doc_id, rev, content, has_conflicts=has_conflicts) - - -COUCH_SCENARIOS = [ - ('couch', {'make_database_for_test': make_couch_database_for_test, - 'copy_database_for_test': copy_couch_database_for_test, - 'make_document_for_test': make_document_for_test, }), -] - - -class CouchTests( - TestWithScenarios, test_backends.AllDatabaseTests, CouchDBTestCase): - - scenarios = COUCH_SCENARIOS - - -class SoledadBackendTests( - TestWithScenarios, - test_backends.LocalDatabaseTests, - CouchDBTestCase): - - scenarios = COUCH_SCENARIOS - - -class CouchValidateGenNTransIdTests( - TestWithScenarios, - test_backends.LocalDatabaseValidateGenNTransIdTests, - CouchDBTestCase): - - scenarios = COUCH_SCENARIOS - - -class CouchValidateSourceGenTests( - TestWithScenarios, - test_backends.LocalDatabaseValidateSourceGenTests, - CouchDBTestCase): - - scenarios = COUCH_SCENARIOS - - -class CouchWithConflictsTests( - TestWithScenarios, - test_backends.LocalDatabaseWithConflictsTests, - CouchDBTestCase): - - scenarios = COUCH_SCENARIOS - - -# Notice: the CouchDB backend does not have indexing capabilities, so we do -# not test indexing now. - -# class CouchIndexTests(test_backends.DatabaseIndexTests, CouchDBTestCase): -# -# scenarios = COUCH_SCENARIOS -# -# def tearDown(self): -# self.db.delete_database() -# test_backends.DatabaseIndexTests.tearDown(self) - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_sync`. -# ----------------------------------------------------------------------------- - -target_scenarios = [ - ('local', {'create_db_and_target': make_local_db_and_target}), ] - - -simple_doc = tests.simple_doc -nested_doc = tests.nested_doc - - -class SoledadBackendSyncTargetTests( - TestWithScenarios, - DatabaseBaseTests, - CouchDBTestCase): - - # TODO: implement _set_trace_hook(_shallow) in CouchSyncTarget so - # skipped tests can be succesfully executed. - - # whitebox true means self.db is the actual local db object - # against which the sync is performed - whitebox = True - - scenarios = (tests.multiply_scenarios(COUCH_SCENARIOS, target_scenarios)) - - def set_trace_hook(self, callback, shallow=False): - setter = (self.st._set_trace_hook if not shallow else - self.st._set_trace_hook_shallow) - try: - setter(callback) - except NotImplementedError: - self.skipTest("%s does not implement _set_trace_hook" - % (self.st.__class__.__name__,)) - - def setUp(self): - CouchDBTestCase.setUp(self) - # other stuff - self.db, self.st = self.create_db_and_target(self) - self.other_changes = [] - - def tearDown(self): - self.db.close() - CouchDBTestCase.tearDown(self) - - def receive_doc(self, doc, gen, trans_id): - self.other_changes.append( - (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) - - def test_sync_exchange_returns_many_new_docs(self): - # This test was replicated to allow dictionaries to be compared after - # JSON expansion (because one dictionary may have many different - # serialized representations). - doc = self.db.create_doc_from_json(simple_doc) - doc2 = self.db.create_doc_from_json(nested_doc) - self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) - new_gen, _ = self.st.sync_exchange( - [], 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) - self.assertEqual(2, new_gen) - self.assertEqual( - [(doc.doc_id, doc.rev, json.loads(simple_doc), 1), - (doc2.doc_id, doc2.rev, json.loads(nested_doc), 2)], - [c[:-3] + (json.loads(c[-3]), c[-2]) for c in self.other_changes]) - if self.whitebox: - self.assertEqual( - self.db._last_exchange_log['return'], - {'last_gen': 2, 'docs': - [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]}) - - def test_get_sync_target(self): - self.assertIsNot(None, self.st) - - def test_get_sync_info(self): - self.assertEqual( - ('test', 0, '', 0, ''), self.st.get_sync_info('other')) - - def test_create_doc_updates_sync_info(self): - self.assertEqual( - ('test', 0, '', 0, ''), self.st.get_sync_info('other')) - self.db.create_doc_from_json(simple_doc) - self.assertEqual(1, self.st.get_sync_info('other')[1]) - - def test_record_sync_info(self): - self.st.record_sync_info('replica', 10, 'T-transid') - self.assertEqual( - ('test', 0, '', 10, 'T-transid'), self.st.get_sync_info('replica')) - - def test_sync_exchange(self): - docs_by_gen = [ - (self.make_document('doc-id', 'replica:1', simple_doc), 10, - 'T-sid')] - new_gen, trans_id = self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertGetDoc(self.db, 'doc-id', 'replica:1', simple_doc, False) - self.assertTransactionLog(['doc-id'], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 1, last_trans_id), - (self.other_changes, new_gen, last_trans_id)) - self.assertEqual(10, self.st.get_sync_info('replica')[3]) - - def test_sync_exchange_deleted(self): - doc = self.db.create_doc_from_json('{}') - edit_rev = 'replica:1|' + doc.rev - docs_by_gen = [ - (self.make_document(doc.doc_id, edit_rev, None), 10, 'T-sid')] - new_gen, trans_id = self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertGetDocIncludeDeleted( - self.db, doc.doc_id, edit_rev, None, False) - self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 2, last_trans_id), - (self.other_changes, new_gen, trans_id)) - self.assertEqual(10, self.st.get_sync_info('replica')[3]) - - def test_sync_exchange_push_many(self): - docs_by_gen = [ - (self.make_document('doc-id', 'replica:1', simple_doc), 10, 'T-1'), - (self.make_document('doc-id2', 'replica:1', nested_doc), 11, - 'T-2')] - new_gen, trans_id = self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertGetDoc(self.db, 'doc-id', 'replica:1', simple_doc, False) - self.assertGetDoc(self.db, 'doc-id2', 'replica:1', nested_doc, False) - self.assertTransactionLog(['doc-id', 'doc-id2'], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 2, last_trans_id), - (self.other_changes, new_gen, trans_id)) - self.assertEqual(11, self.st.get_sync_info('replica')[3]) - - def test_sync_exchange_refuses_conflicts(self): - doc = self.db.create_doc_from_json(simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_doc = '{"key": "altval"}' - docs_by_gen = [ - (self.make_document(doc.doc_id, 'replica:1', new_doc), 10, - 'T-sid')] - new_gen, _ = self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id], self.db) - self.assertEqual( - (doc.doc_id, doc.rev, simple_doc, 1), self.other_changes[0][:-1]) - self.assertEqual(1, new_gen) - if self.whitebox: - self.assertEqual(self.db._last_exchange_log['return'], - {'last_gen': 1, 'docs': [(doc.doc_id, doc.rev)]}) - - def test_sync_exchange_ignores_convergence(self): - doc = self.db.create_doc_from_json(simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - gen, txid = self.db._get_generation_info() - docs_by_gen = [ - (self.make_document(doc.doc_id, doc.rev, simple_doc), 10, 'T-sid')] - new_gen, _ = self.st.sync_exchange( - docs_by_gen, 'replica', last_known_generation=gen, - last_known_trans_id=txid, return_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id], self.db) - self.assertEqual(([], 1), (self.other_changes, new_gen)) - - def test_sync_exchange_returns_new_docs(self): - doc = self.db.create_doc_from_json(simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_gen, _ = self.st.sync_exchange( - [], 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id], self.db) - self.assertEqual( - (doc.doc_id, doc.rev, simple_doc, 1), self.other_changes[0][:-1]) - self.assertEqual(1, new_gen) - if self.whitebox: - self.assertEqual(self.db._last_exchange_log['return'], - {'last_gen': 1, 'docs': [(doc.doc_id, doc.rev)]}) - - def test_sync_exchange_returns_deleted_docs(self): - doc = self.db.create_doc_from_json(simple_doc) - self.db.delete_doc(doc) - self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) - new_gen, _ = self.st.sync_exchange( - [], 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) - self.assertEqual( - (doc.doc_id, doc.rev, None, 2), self.other_changes[0][:-1]) - self.assertEqual(2, new_gen) - if self.whitebox: - self.assertEqual(self.db._last_exchange_log['return'], - {'last_gen': 2, 'docs': [(doc.doc_id, doc.rev)]}) - - def test_sync_exchange_getting_newer_docs(self): - doc = self.db.create_doc_from_json(simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_doc = '{"key": "altval"}' - docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, - 'T-sid')] - new_gen, _ = self.st.sync_exchange( - docs_by_gen, 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) - self.assertEqual(([], 2), (self.other_changes, new_gen)) - - def test_sync_exchange_with_concurrent_updates_of_synced_doc(self): - expected = [] - - def before_whatschanged_cb(state): - if state != 'before whats_changed': - return - cont = '{"key": "cuncurrent"}' - conc_rev = self.db.put_doc( - self.make_document(doc.doc_id, 'test:1|z:2', cont)) - expected.append((doc.doc_id, conc_rev, cont, 3)) - - self.set_trace_hook(before_whatschanged_cb) - doc = self.db.create_doc_from_json(simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_doc = '{"key": "altval"}' - docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, - 'T-sid')] - new_gen, _ = self.st.sync_exchange( - docs_by_gen, 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertEqual(expected, [c[:-1] for c in self.other_changes]) - self.assertEqual(3, new_gen) - - def test_sync_exchange_with_concurrent_updates(self): - - def after_whatschanged_cb(state): - if state != 'after whats_changed': - return - self.db.create_doc_from_json('{"new": "doc"}') - - self.set_trace_hook(after_whatschanged_cb) - doc = self.db.create_doc_from_json(simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - new_doc = '{"key": "altval"}' - docs_by_gen = [ - (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, - 'T-sid')] - new_gen, _ = self.st.sync_exchange( - docs_by_gen, 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertEqual(([], 2), (self.other_changes, new_gen)) - - def test_sync_exchange_converged_handling(self): - doc = self.db.create_doc_from_json(simple_doc) - docs_by_gen = [ - (self.make_document('new', 'other:1', '{}'), 4, 'T-foo'), - (self.make_document(doc.doc_id, doc.rev, doc.get_json()), 5, - 'T-bar')] - new_gen, _ = self.st.sync_exchange( - docs_by_gen, 'other-replica', last_known_generation=0, - last_known_trans_id=None, return_doc_cb=self.receive_doc) - self.assertEqual(([], 2), (self.other_changes, new_gen)) - - def test_sync_exchange_detect_incomplete_exchange(self): - def before_get_docs_explode(state): - if state != 'before get_docs': - return - raise u1db_errors.U1DBError("fail") - self.set_trace_hook(before_get_docs_explode) - # suppress traceback printing in the wsgiref server - # self.patch(simple_server.ServerHandler, - # 'log_exception', lambda h, exc_info: None) - doc = self.db.create_doc_from_json(simple_doc) - self.assertTransactionLog([doc.doc_id], self.db) - self.assertRaises( - (u1db_errors.U1DBError, u1db_errors.BrokenSyncStream), - self.st.sync_exchange, [], 'other-replica', - last_known_generation=0, last_known_trans_id=None, - return_doc_cb=self.receive_doc) - - def test_sync_exchange_doc_ids(self): - sync_exchange_doc_ids = getattr(self.st, 'sync_exchange_doc_ids', None) - if sync_exchange_doc_ids is None: - self.skipTest("sync_exchange_doc_ids not implemented") - db2 = self.create_database('test2') - doc = db2.create_doc_from_json(simple_doc) - new_gen, trans_id = sync_exchange_doc_ids( - db2, [(doc.doc_id, 10, 'T-sid')], 0, None, - return_doc_cb=self.receive_doc) - self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) - self.assertTransactionLog([doc.doc_id], self.db) - last_trans_id = self.getLastTransId(self.db) - self.assertEqual(([], 1, last_trans_id), - (self.other_changes, new_gen, trans_id)) - self.assertEqual(10, self.st.get_sync_info(db2._replica_uid)[3]) - - def test__set_trace_hook(self): - called = [] - - def cb(state): - called.append(state) - - self.set_trace_hook(cb) - self.st.sync_exchange([], 'replica', 0, None, self.receive_doc) - self.st.record_sync_info('replica', 0, 'T-sid') - self.assertEqual(['before whats_changed', - 'after whats_changed', - 'before get_docs', - 'record_sync_info', - ], - called) - - def test__set_trace_hook_shallow(self): - st_trace_shallow = self.st._set_trace_hook_shallow - target_st_trace_shallow = SyncTarget._set_trace_hook_shallow - same_meth = st_trace_shallow == self.st._set_trace_hook - same_fun = st_trace_shallow.im_func == target_st_trace_shallow.im_func - if (same_meth or same_fun): - # shallow same as full - expected = ['before whats_changed', - 'after whats_changed', - 'before get_docs', - 'record_sync_info', - ] - else: - expected = ['sync_exchange', 'record_sync_info'] - - called = [] - - def cb(state): - called.append(state) - - self.set_trace_hook(cb, shallow=True) - self.st.sync_exchange([], 'replica', 0, None, self.receive_doc) - self.st.record_sync_info('replica', 0, 'T-sid') - self.assertEqual(expected, called) - -sync_scenarios = [] -for name, scenario in COUCH_SCENARIOS: - scenario = dict(scenario) - scenario['do_sync'] = sync_via_synchronizer - sync_scenarios.append((name, scenario)) - scenario = dict(scenario) - - -class SoledadBackendSyncTests( - TestWithScenarios, - DatabaseBaseTests, - CouchDBTestCase): - - scenarios = sync_scenarios - - def create_database(self, replica_uid, sync_role=None): - if replica_uid == 'test' and sync_role is None: - # created up the chain by base class but unused - return None - db = self.create_database_for_role(replica_uid, sync_role) - if sync_role: - self._use_tracking[db] = (replica_uid, sync_role) - return db - - def create_database_for_role(self, replica_uid, sync_role): - # hook point for reuse - return DatabaseBaseTests.create_database(self, replica_uid) - - def copy_database(self, db, sync_role=None): - # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES - # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST - # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS - # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND - # NINJA TO YOUR HOUSE. - db_copy = self.copy_database_for_test(self, db) - name, orig_sync_role = self._use_tracking[db] - self._use_tracking[db_copy] = ( - name + '(copy)', sync_role or orig_sync_role) - return db_copy - - def sync(self, db_from, db_to, trace_hook=None, - trace_hook_shallow=None): - from_name, from_sync_role = self._use_tracking[db_from] - to_name, to_sync_role = self._use_tracking[db_to] - if from_sync_role not in ('source', 'both'): - raise Exception("%s marked for %s use but used as source" % - (from_name, from_sync_role)) - if to_sync_role not in ('target', 'both'): - raise Exception("%s marked for %s use but used as target" % - (to_name, to_sync_role)) - return self.do_sync(self, db_from, db_to, trace_hook, - trace_hook_shallow) - - def setUp(self): - self.db = None - self.db1 = None - self.db2 = None - self.db3 = None - self.db1_copy = None - self.db2_copy = None - self._use_tracking = {} - DatabaseBaseTests.setUp(self) - - def tearDown(self): - for db in [ - self.db, self.db1, self.db2, - self.db3, self.db1_copy, self.db2_copy - ]: - if db is not None: - self.delete_db(db._dbname) - db.close() - DatabaseBaseTests.tearDown(self) - - def assertLastExchangeLog(self, db, expected): - log = getattr(db, '_last_exchange_log', None) - if log is None: - return - self.assertEqual(expected, log) - - def test_sync_tracks_db_generation_of_other(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.assertEqual(0, self.sync(self.db1, self.db2)) - self.assertEqual( - (0, ''), self.db1._get_replica_gen_and_trans_id('test2')) - self.assertEqual( - (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [], 'last_known_gen': 0}, - 'return': {'docs': [], 'last_gen': 0}}) - - def test_sync_autoresolves(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc1 = self.db1.create_doc_from_json(simple_doc, doc_id='doc') - rev1 = doc1.rev - doc2 = self.db2.create_doc_from_json(simple_doc, doc_id='doc') - rev2 = doc2.rev - self.sync(self.db1, self.db2) - doc = self.db1.get_doc('doc') - self.assertFalse(doc.has_conflicts) - self.assertEqual(doc.rev, self.db2.get_doc('doc').rev) - v = vectorclock.VectorClockRev(doc.rev) - self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev1))) - self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev2))) - - def test_sync_autoresolves_moar(self): - # here we test that when a database that has a conflicted document is - # the source of a sync, and the target database has a revision of the - # conflicted document that is newer than the source database's, and - # that target's database's document's content is the same as the - # source's document's conflict's, the source's document's conflict gets - # autoresolved, and the source's document's revision bumped. - # - # idea is as follows: - # A B - # a1 - - # `-------> - # a1 a1 - # v v - # a2 a1b1 - # `-------> - # a1b1+a2 a1b1 - # v - # a1b1+a2 a1b2 (a1b2 has same content as a2) - # `-------> - # a3b2 a1b2 (autoresolved) - # `-------> - # a3b2 a3b2 - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(simple_doc, doc_id='doc') - self.sync(self.db1, self.db2) - for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: - doc = db.get_doc('doc') - doc.set_json(content) - db.put_doc(doc) - self.sync(self.db1, self.db2) - # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict - doc = self.db1.get_doc('doc') - rev1 = doc.rev - self.assertTrue(doc.has_conflicts) - # set db2 to have a doc of {} (same as db1 before the conflict) - doc = self.db2.get_doc('doc') - doc.set_json('{}') - self.db2.put_doc(doc) - rev2 = doc.rev - # sync it across - self.sync(self.db1, self.db2) - # tadaa! - doc = self.db1.get_doc('doc') - self.assertFalse(doc.has_conflicts) - vec1 = vectorclock.VectorClockRev(rev1) - vec2 = vectorclock.VectorClockRev(rev2) - vec3 = vectorclock.VectorClockRev(doc.rev) - self.assertTrue(vec3.is_newer(vec1)) - self.assertTrue(vec3.is_newer(vec2)) - # because the conflict is on the source, sync it another time - self.sync(self.db1, self.db2) - # make sure db2 now has the exact same thing - self.assertEqual(self.db1.get_doc('doc'), self.db2.get_doc('doc')) - - def test_sync_autoresolves_moar_backwards(self): - # here we test that when a database that has a conflicted document is - # the target of a sync, and the source database has a revision of the - # conflicted document that is newer than the target database's, and - # that source's database's document's content is the same as the - # target's document's conflict's, the target's document's conflict gets - # autoresolved, and the document's revision bumped. - # - # idea is as follows: - # A B - # a1 - - # `-------> - # a1 a1 - # v v - # a2 a1b1 - # `-------> - # a1b1+a2 a1b1 - # v - # a1b1+a2 a1b2 (a1b2 has same content as a2) - # <-------' - # a3b2 a3b2 (autoresolved and propagated) - self.db1 = self.create_database('test1', 'both') - self.db2 = self.create_database('test2', 'both') - self.db1.create_doc_from_json(simple_doc, doc_id='doc') - self.sync(self.db1, self.db2) - for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: - doc = db.get_doc('doc') - doc.set_json(content) - db.put_doc(doc) - self.sync(self.db1, self.db2) - # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict - doc = self.db1.get_doc('doc') - rev1 = doc.rev - self.assertTrue(doc.has_conflicts) - revc = self.db1.get_doc_conflicts('doc')[-1].rev - # set db2 to have a doc of {} (same as db1 before the conflict) - doc = self.db2.get_doc('doc') - doc.set_json('{}') - self.db2.put_doc(doc) - rev2 = doc.rev - # sync it across - self.sync(self.db2, self.db1) - # tadaa! - doc = self.db1.get_doc('doc') - self.assertFalse(doc.has_conflicts) - vec1 = vectorclock.VectorClockRev(rev1) - vec2 = vectorclock.VectorClockRev(rev2) - vec3 = vectorclock.VectorClockRev(doc.rev) - vecc = vectorclock.VectorClockRev(revc) - self.assertTrue(vec3.is_newer(vec1)) - self.assertTrue(vec3.is_newer(vec2)) - self.assertTrue(vec3.is_newer(vecc)) - # make sure db2 now has the exact same thing - self.assertEqual(self.db1.get_doc('doc'), self.db2.get_doc('doc')) - - def test_sync_autoresolves_moar_backwards_three(self): - # same as autoresolves_moar_backwards, but with three databases (note - # all the syncs go in the same direction -- this is a more natural - # scenario): - # - # A B C - # a1 - - - # `-------> - # a1 a1 - - # `-------> - # a1 a1 a1 - # v v - # a2 a1b1 a1 - # `-------------------> - # a2 a1b1 a2 - # `-------> - # a2+a1b1 a2 - # v - # a2 a2+a1b1 a2c1 (same as a1b1) - # `-------------------> - # a2c1 a2+a1b1 a2c1 - # `-------> - # a2b2c1 a2b2c1 a2c1 - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'both') - self.db3 = self.create_database('test3', 'target') - self.db1.create_doc_from_json(simple_doc, doc_id='doc') - self.sync(self.db1, self.db2) - self.sync(self.db2, self.db3) - for db, content in [(self.db2, '{"hi": 42}'), - (self.db1, '{}'), - ]: - doc = db.get_doc('doc') - doc.set_json(content) - db.put_doc(doc) - self.sync(self.db1, self.db3) - self.sync(self.db2, self.db3) - # db2 and db3 now both have a doc of {}, but db2 has a - # conflict - doc = self.db2.get_doc('doc') - self.assertTrue(doc.has_conflicts) - revc = self.db2.get_doc_conflicts('doc')[-1].rev - self.assertEqual('{}', doc.get_json()) - self.assertEqual(self.db3.get_doc('doc').get_json(), doc.get_json()) - self.assertEqual(self.db3.get_doc('doc').rev, doc.rev) - # set db3 to have a doc of {hi:42} (same as db2 before the conflict) - doc = self.db3.get_doc('doc') - doc.set_json('{"hi": 42}') - self.db3.put_doc(doc) - rev3 = doc.rev - # sync it across to db1 - self.sync(self.db1, self.db3) - # db1 now has hi:42, with a rev that is newer than db2's doc - doc = self.db1.get_doc('doc') - rev1 = doc.rev - self.assertFalse(doc.has_conflicts) - self.assertEqual('{"hi": 42}', doc.get_json()) - VCR = vectorclock.VectorClockRev - self.assertTrue(VCR(rev1).is_newer(VCR(self.db2.get_doc('doc').rev))) - # so sync it to db2 - self.sync(self.db1, self.db2) - # tadaa! - doc = self.db2.get_doc('doc') - self.assertFalse(doc.has_conflicts) - # db2's revision of the document is strictly newer than db1's before - # the sync, and db3's before that sync way back when - self.assertTrue(VCR(doc.rev).is_newer(VCR(rev1))) - self.assertTrue(VCR(doc.rev).is_newer(VCR(rev3))) - self.assertTrue(VCR(doc.rev).is_newer(VCR(revc))) - # make sure both dbs now have the exact same thing - self.assertEqual(self.db1.get_doc('doc'), self.db2.get_doc('doc')) - - def test_sync_puts_changes(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db1.create_doc_from_json(simple_doc) - self.assertEqual(1, self.sync(self.db1, self.db2)) - self.assertGetDoc(self.db2, doc.doc_id, doc.rev, simple_doc, False) - self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) - self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [(doc.doc_id, doc.rev)], - 'source_uid': 'test1', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': {'docs': [], 'last_gen': 1}}) - - def test_sync_pulls_changes(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db2.create_doc_from_json(simple_doc) - self.assertEqual(0, self.sync(self.db1, self.db2)) - self.assertGetDoc(self.db1, doc.doc_id, doc.rev, simple_doc, False) - self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) - self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [], 'last_known_gen': 0}, - 'return': {'docs': [(doc.doc_id, doc.rev)], - 'last_gen': 1}}) - self.assertGetDoc(self.db2, doc.doc_id, doc.rev, simple_doc, False) - - def test_sync_pulling_doesnt_update_other_if_changed(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db2.create_doc_from_json(simple_doc) - # After the local side has sent its list of docs, before we start - # receiving the "targets" response, we update the local database with a - # new record. - # When we finish synchronizing, we can notice that something locally - # was updated, and we cannot tell c2 our new updated generation - - def before_get_docs(state): - if state != 'before get_docs': - return - self.db1.create_doc_from_json(simple_doc) - - self.assertEqual(0, self.sync(self.db1, self.db2, - trace_hook=before_get_docs)) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [], 'last_known_gen': 0}, - 'return': {'docs': [(doc.doc_id, doc.rev)], - 'last_gen': 1}}) - self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) - # c2 should not have gotten a '_record_sync_info' call, because the - # local database had been updated more than just by the messages - # returned from c2. - self.assertEqual( - (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) - - def test_sync_doesnt_update_other_if_nothing_pulled(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(simple_doc) - - def no_record_sync_info(state): - if state != 'record_sync_info': - return - self.fail('SyncTarget.record_sync_info was called') - self.assertEqual(1, self.sync(self.db1, self.db2, - trace_hook_shallow=no_record_sync_info)) - self.assertEqual( - 1, - self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)[0]) - - def test_sync_ignores_convergence(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'both') - doc = self.db1.create_doc_from_json(simple_doc) - self.db3 = self.create_database('test3', 'target') - self.assertEqual(1, self.sync(self.db1, self.db3)) - self.assertEqual(0, self.sync(self.db2, self.db3)) - self.assertEqual(1, self.sync(self.db1, self.db2)) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [(doc.doc_id, doc.rev)], - 'source_uid': 'test1', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': {'docs': [], 'last_gen': 1}}) - - def test_sync_ignores_superseded(self): - self.db1 = self.create_database('test1', 'both') - self.db2 = self.create_database('test2', 'both') - doc = self.db1.create_doc_from_json(simple_doc) - doc_rev1 = doc.rev - self.db3 = self.create_database('test3', 'target') - self.sync(self.db1, self.db3) - self.sync(self.db2, self.db3) - new_content = '{"key": "altval"}' - doc.set_json(new_content) - self.db1.put_doc(doc) - doc_rev2 = doc.rev - self.sync(self.db2, self.db1) - self.assertLastExchangeLog( - self.db1, - {'receive': {'docs': [(doc.doc_id, doc_rev1)], - 'source_uid': 'test2', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': {'docs': [(doc.doc_id, doc_rev2)], - 'last_gen': 2}}) - self.assertGetDoc(self.db1, doc.doc_id, doc_rev2, new_content, False) - - def test_sync_sees_remote_conflicted(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc1 = self.db1.create_doc_from_json(simple_doc) - doc_id = doc1.doc_id - doc1_rev = doc1.rev - new_doc = '{"key": "altval"}' - doc2 = self.db2.create_doc_from_json(new_doc, doc_id=doc_id) - doc2_rev = doc2.rev - self.assertTransactionLog([doc1.doc_id], self.db1) - self.sync(self.db1, self.db2) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [(doc_id, doc1_rev)], - 'source_uid': 'test1', - 'source_gen': 1, 'last_known_gen': 0}, - 'return': {'docs': [(doc_id, doc2_rev)], - 'last_gen': 1}}) - self.assertTransactionLog([doc_id, doc_id], self.db1) - self.assertGetDoc(self.db1, doc_id, doc2_rev, new_doc, True) - self.assertGetDoc(self.db2, doc_id, doc2_rev, new_doc, False) - - def test_sync_sees_remote_delete_conflicted(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc1 = self.db1.create_doc_from_json(simple_doc) - doc_id = doc1.doc_id - self.sync(self.db1, self.db2) - doc2 = self.make_document(doc1.doc_id, doc1.rev, doc1.get_json()) - new_doc = '{"key": "altval"}' - doc1.set_json(new_doc) - self.db1.put_doc(doc1) - self.db2.delete_doc(doc2) - self.assertTransactionLog([doc_id, doc_id], self.db1) - self.sync(self.db1, self.db2) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [(doc_id, doc1.rev)], - 'source_uid': 'test1', - 'source_gen': 2, 'last_known_gen': 1}, - 'return': {'docs': [(doc_id, doc2.rev)], - 'last_gen': 2}}) - self.assertTransactionLog([doc_id, doc_id, doc_id], self.db1) - self.assertGetDocIncludeDeleted(self.db1, doc_id, doc2.rev, None, True) - self.assertGetDocIncludeDeleted( - self.db2, doc_id, doc2.rev, None, False) - - def test_sync_local_race_conflicted(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - doc = self.db1.create_doc_from_json(simple_doc) - doc_id = doc.doc_id - doc1_rev = doc.rev - self.sync(self.db1, self.db2) - content1 = '{"key": "localval"}' - content2 = '{"key": "altval"}' - doc.set_json(content2) - self.db2.put_doc(doc) - doc2_rev2 = doc.rev - triggered = [] - - def after_whatschanged(state): - if state != 'after whats_changed': - return - triggered.append(True) - doc = self.make_document(doc_id, doc1_rev, content1) - self.db1.put_doc(doc) - - self.sync(self.db1, self.db2, trace_hook=after_whatschanged) - self.assertEqual([True], triggered) - self.assertGetDoc(self.db1, doc_id, doc2_rev2, content2, True) - - def test_sync_propagates_deletes(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'both') - doc1 = self.db1.create_doc_from_json(simple_doc) - doc_id = doc1.doc_id - self.sync(self.db1, self.db2) - self.db3 = self.create_database('test3', 'target') - self.sync(self.db1, self.db3) - self.db1.delete_doc(doc1) - deleted_rev = doc1.rev - self.sync(self.db1, self.db2) - self.assertLastExchangeLog( - self.db2, - {'receive': {'docs': [(doc_id, deleted_rev)], - 'source_uid': 'test1', - 'source_gen': 2, 'last_known_gen': 1}, - 'return': {'docs': [], 'last_gen': 2}}) - self.assertGetDocIncludeDeleted( - self.db1, doc_id, deleted_rev, None, False) - self.assertGetDocIncludeDeleted( - self.db2, doc_id, deleted_rev, None, False) - self.sync(self.db2, self.db3) - self.assertLastExchangeLog( - self.db3, - {'receive': {'docs': [(doc_id, deleted_rev)], - 'source_uid': 'test2', - 'source_gen': 2, 'last_known_gen': 0}, - 'return': {'docs': [], 'last_gen': 2}}) - self.assertGetDocIncludeDeleted( - self.db3, doc_id, deleted_rev, None, False) - - def test_sync_propagates_deletes_2(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json('{"a": "1"}', doc_id='the-doc') - self.sync(self.db1, self.db2) - doc1_2 = self.db2.get_doc('the-doc') - self.db2.delete_doc(doc1_2) - self.sync(self.db1, self.db2) - self.assertGetDocIncludeDeleted( - self.db1, 'the-doc', doc1_2.rev, None, False) - - def test_sync_propagates_resolution(self): - self.db1 = self.create_database('test1', 'both') - self.db2 = self.create_database('test2', 'both') - doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') - self.db3 = self.create_database('test3', 'both') - self.sync(self.db2, self.db1) - self.assertEqual( - self.db1._get_generation_info(), - self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)) - self.assertEqual( - self.db2._get_generation_info(), - self.db1._get_replica_gen_and_trans_id(self.db2._replica_uid)) - self.sync(self.db3, self.db1) - # update on 2 - doc2 = self.make_document('the-doc', doc1.rev, '{"a": 2}') - self.db2.put_doc(doc2) - self.sync(self.db2, self.db3) - self.assertEqual(self.db3.get_doc('the-doc').rev, doc2.rev) - # update on 1 - doc1.set_json('{"a": 3}') - self.db1.put_doc(doc1) - # conflicts - self.sync(self.db2, self.db1) - self.sync(self.db3, self.db1) - self.assertTrue(self.db2.get_doc('the-doc').has_conflicts) - self.assertTrue(self.db3.get_doc('the-doc').has_conflicts) - # resolve - conflicts = self.db2.get_doc_conflicts('the-doc') - doc4 = self.make_document('the-doc', None, '{"a": 4}') - revs = [doc.rev for doc in conflicts] - self.db2.resolve_doc(doc4, revs) - doc2 = self.db2.get_doc('the-doc') - self.assertEqual(doc4.get_json(), doc2.get_json()) - self.assertFalse(doc2.has_conflicts) - self.sync(self.db2, self.db3) - doc3 = self.db3.get_doc('the-doc') - self.assertEqual(doc4.get_json(), doc3.get_json()) - self.assertFalse(doc3.has_conflicts) - - def test_sync_supersedes_conflicts(self): - self.db1 = self.create_database('test1', 'both') - self.db2 = self.create_database('test2', 'target') - self.db3 = self.create_database('test3', 'both') - doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') - self.db2.create_doc_from_json('{"b": 1}', doc_id='the-doc') - self.db3.create_doc_from_json('{"c": 1}', doc_id='the-doc') - self.sync(self.db3, self.db1) - self.assertEqual( - self.db1._get_generation_info(), - self.db3._get_replica_gen_and_trans_id(self.db1._replica_uid)) - self.assertEqual( - self.db3._get_generation_info(), - self.db1._get_replica_gen_and_trans_id(self.db3._replica_uid)) - self.sync(self.db3, self.db2) - self.assertEqual( - self.db2._get_generation_info(), - self.db3._get_replica_gen_and_trans_id(self.db2._replica_uid)) - self.assertEqual( - self.db3._get_generation_info(), - self.db2._get_replica_gen_and_trans_id(self.db3._replica_uid)) - self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) - doc1.set_json('{"a": 2}') - self.db1.put_doc(doc1) - self.sync(self.db3, self.db1) - # original doc1 should have been removed from conflicts - self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) - - def test_sync_stops_after_get_sync_info(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc) - self.sync(self.db1, self.db2) - - def put_hook(state): - self.fail("Tracehook triggered for %s" % (state,)) - - self.sync(self.db1, self.db2, trace_hook_shallow=put_hook) - - def test_sync_detects_identical_replica_uid(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test1', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') - self.assertRaises( - u1db_errors.InvalidReplicaUID, self.sync, self.db1, self.db2) - - def test_sync_detects_rollback_in_source(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') - self.sync(self.db1, self.db2) - self.db1_copy = self.copy_database(self.db1) - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.sync(self.db1, self.db2) - self.assertRaises( - u1db_errors.InvalidGeneration, self.sync, self.db1_copy, self.db2) - - def test_sync_detects_rollback_in_target(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.db2_copy = self.copy_database(self.db2) - self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.sync(self.db1, self.db2) - self.assertRaises( - u1db_errors.InvalidGeneration, self.sync, self.db1, self.db2_copy) - - def test_sync_detects_diverged_source(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db3 = self.copy_database(self.db1) - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.db3.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.assertRaises( - u1db_errors.InvalidTransactionId, self.sync, self.db3, self.db2) - - def test_sync_detects_diverged_target(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db3 = self.copy_database(self.db2) - self.db3.create_doc_from_json(tests.nested_doc, doc_id="divergent") - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.assertRaises( - u1db_errors.InvalidTransactionId, self.sync, self.db1, self.db3) - - def test_sync_detects_rollback_and_divergence_in_source(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') - self.sync(self.db1, self.db2) - self.db1_copy = self.copy_database(self.db1) - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.sync(self.db1, self.db2) - self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.assertRaises( - u1db_errors.InvalidTransactionId, self.sync, - self.db1_copy, self.db2) - - def test_sync_detects_rollback_and_divergence_in_target(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") - self.sync(self.db1, self.db2) - self.db2_copy = self.copy_database(self.db2) - self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.sync(self.db1, self.db2) - self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') - self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') - self.assertRaises( - u1db_errors.InvalidTransactionId, self.sync, - self.db1, self.db2_copy) - - def test_optional_sync_preserve_json(self): - self.db1 = self.create_database('test1', 'source') - self.db2 = self.create_database('test2', 'target') - cont1 = '{"a": 2}' - cont2 = '{"b": 3}' - self.db1.create_doc_from_json(cont1, doc_id="1") - self.db2.create_doc_from_json(cont2, doc_id="2") - self.sync(self.db1, self.db2) - self.assertEqual(cont1, self.db2.get_doc("1").get_json()) - self.assertEqual(cont2, self.db1.get_doc("2").get_json()) - - -class SoledadBackendExceptionsTests(CouchDBTestCase): - - def setUp(self): - CouchDBTestCase.setUp(self) - - def create_db(self, ensure=True, dbname=None): - if not dbname: - dbname = ('test-%s' % uuid4().hex) - if dbname not in self.couch_server: - self.couch_server.create(dbname) - self.db = couch.CouchDatabase( - ('http://127.0.0.1:%d' % self.couch_port), - dbname, - ensure_ddocs=ensure) - - def tearDown(self): - self.db.delete_database() - self.db.close() - CouchDBTestCase.tearDown(self) - - def test_missing_design_doc_raises(self): - """ - Test that all methods that access design documents will raise if the - design docs are not present. - """ - self.create_db(ensure=False) - # get_generation_info() - self.assertRaises( - errors.MissingDesignDocError, - self.db.get_generation_info) - # get_trans_id_for_gen() - self.assertRaises( - errors.MissingDesignDocError, - self.db.get_trans_id_for_gen, 1) - # get_transaction_log() - self.assertRaises( - errors.MissingDesignDocError, - self.db.get_transaction_log) - # whats_changed() - self.assertRaises( - errors.MissingDesignDocError, - self.db.whats_changed) - - def test_missing_design_doc_functions_raises(self): - """ - Test that all methods that access design documents list functions - will raise if the functions are not present. - """ - self.create_db(ensure=True) - # erase views from _design/transactions - transactions = self.db._database['_design/transactions'] - transactions['lists'] = {} - self.db._database.save(transactions) - # get_generation_info() - self.assertRaises( - errors.MissingDesignDocListFunctionError, - self.db.get_generation_info) - # get_trans_id_for_gen() - self.assertRaises( - errors.MissingDesignDocListFunctionError, - self.db.get_trans_id_for_gen, 1) - # whats_changed() - self.assertRaises( - errors.MissingDesignDocListFunctionError, - self.db.whats_changed) - - def test_absent_design_doc_functions_raises(self): - """ - Test that all methods that access design documents list functions - will raise if the functions are not present. - """ - self.create_db(ensure=True) - # erase views from _design/transactions - transactions = self.db._database['_design/transactions'] - del transactions['lists'] - self.db._database.save(transactions) - # get_generation_info() - self.assertRaises( - errors.MissingDesignDocListFunctionError, - self.db.get_generation_info) - # _get_trans_id_for_gen() - self.assertRaises( - errors.MissingDesignDocListFunctionError, - self.db.get_trans_id_for_gen, 1) - # whats_changed() - self.assertRaises( - errors.MissingDesignDocListFunctionError, - self.db.whats_changed) - - def test_missing_design_doc_named_views_raises(self): - """ - Test that all methods that access design documents' named views will - raise if the views are not present. - """ - self.create_db(ensure=True) - # erase views from _design/docs - docs = self.db._database['_design/docs'] - del docs['views'] - self.db._database.save(docs) - # erase views from _design/syncs - syncs = self.db._database['_design/syncs'] - del syncs['views'] - self.db._database.save(syncs) - # erase views from _design/transactions - transactions = self.db._database['_design/transactions'] - del transactions['views'] - self.db._database.save(transactions) - # get_generation_info() - self.assertRaises( - errors.MissingDesignDocNamedViewError, - self.db.get_generation_info) - # _get_trans_id_for_gen() - self.assertRaises( - errors.MissingDesignDocNamedViewError, - self.db.get_trans_id_for_gen, 1) - # _get_transaction_log() - self.assertRaises( - errors.MissingDesignDocNamedViewError, - self.db.get_transaction_log) - # whats_changed() - self.assertRaises( - errors.MissingDesignDocNamedViewError, - self.db.whats_changed) - - def test_deleted_design_doc_raises(self): - """ - Test that all methods that access design documents will raise if the - design docs are not present. - """ - self.create_db(ensure=True) - # delete _design/docs - del self.db._database['_design/docs'] - # delete _design/syncs - del self.db._database['_design/syncs'] - # delete _design/transactions - del self.db._database['_design/transactions'] - # get_generation_info() - self.assertRaises( - errors.MissingDesignDocDeletedError, - self.db.get_generation_info) - # get_trans_id_for_gen() - self.assertRaises( - errors.MissingDesignDocDeletedError, - self.db.get_trans_id_for_gen, 1) - # get_transaction_log() - self.assertRaises( - errors.MissingDesignDocDeletedError, - self.db.get_transaction_log) - # whats_changed() - self.assertRaises( - errors.MissingDesignDocDeletedError, - self.db.whats_changed) - - def test_ensure_ddoc_independently(self): - """ - Test that a missing ddocs other than _design/docs will be ensured - even if _design/docs is there. - """ - self.create_db(ensure=True) - del self.db._database['_design/transactions'] - self.assertRaises( - errors.MissingDesignDocDeletedError, - self.db.get_transaction_log) - self.create_db(ensure=True, dbname=self.db._dbname) - self.db.get_transaction_log() - - def test_ensure_security_doc(self): - """ - Ensure_security creates a _security ddoc to ensure that only soledad - will have the lowest privileged access to an user db. - """ - self.create_db(ensure=False) - self.assertFalse(self.db._database.resource.get_json('_security')[2]) - self.db.ensure_security_ddoc() - security_ddoc = self.db._database.resource.get_json('_security')[2] - self.assertIn('admins', security_ddoc) - self.assertFalse(security_ddoc['admins']['names']) - self.assertIn('members', security_ddoc) - self.assertIn('soledad', security_ddoc['members']['names']) - - def test_ensure_security_from_configuration(self): - """ - Given a configuration, follow it to create the security document - """ - self.create_db(ensure=False) - configuration = {'members': ['user1', 'user2'], - 'members_roles': ['role1', 'role2'], - 'admins': ['admin'], - 'admins_roles': ['administrators'] - } - self.db.ensure_security_ddoc(configuration) - - security_ddoc = self.db._database.resource.get_json('_security')[2] - self.assertEquals(configuration['admins'], - security_ddoc['admins']['names']) - self.assertEquals(configuration['admins_roles'], - security_ddoc['admins']['roles']) - self.assertEquals(configuration['members'], - security_ddoc['members']['names']) - self.assertEquals(configuration['members_roles'], - security_ddoc['members']['roles']) - - -class DatabaseNameValidationTest(unittest.TestCase): - - def test_database_name_validation(self): - inject = couch.state.is_db_name_valid("user-deadbeef | cat /secret") - self.assertFalse(inject) - self.assertTrue(couch.state.is_db_name_valid("user-cafe1337")) - - -class CommandBasedDBCreationTest(unittest.TestCase): - - def test_ensure_db_using_custom_command(self): - state = couch.state.CouchServerState("url", create_cmd="echo") - mock_db = Mock() - mock_db.replica_uid = 'replica_uid' - state.open_database = Mock(return_value=mock_db) - db, replica_uid = state.ensure_database("user-1337") # works - self.assertEquals(mock_db, db) - self.assertEquals(mock_db.replica_uid, replica_uid) - - def test_raises_unauthorized_on_failure(self): - state = couch.state.CouchServerState("url", create_cmd="inexistent") - self.assertRaises(u1db_errors.Unauthorized, - state.ensure_database, "user-1337") - - def test_raises_unauthorized_by_default(self): - state = couch.state.CouchServerState("url") - self.assertRaises(u1db_errors.Unauthorized, - state.ensure_database, "user-1337") diff --git a/testing/tests/couch/test_couch_operations_atomicity.py b/testing/tests/couch/test_couch_operations_atomicity.py deleted file mode 100644 index aec9c6cf..00000000 --- a/testing/tests/couch/test_couch_operations_atomicity.py +++ /dev/null @@ -1,371 +0,0 @@ -# -*- coding: utf-8 -*- -# test_couch_operations_atomicity.py -# Copyright (C) 2013, 2014 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Test atomicity of couch operations. -""" -import os -import tempfile -import threading - -from urlparse import urljoin -from twisted.internet import defer -from uuid import uuid4 - -from leap.soledad.client import Soledad -from leap.soledad.common.couch.state import CouchServerState -from leap.soledad.common.couch import CouchDatabase - -from test_soledad.util import ( - make_token_soledad_app, - make_soledad_document_for_test, - soledad_sync_target, -) -from test_soledad.util import CouchDBTestCase -from test_soledad.u1db_tests import TestCaseWithServer - - -REPEAT_TIMES = 20 - - -class CouchAtomicityTestCase(CouchDBTestCase, TestCaseWithServer): - - @staticmethod - def make_app_after_state(state): - return make_token_soledad_app(state) - - make_document_for_test = make_soledad_document_for_test - - sync_target = soledad_sync_target - - def _soledad_instance(self, user=None, passphrase=u'123', - prefix='', - secrets_path='secrets.json', - local_db_path='soledad.u1db', server_url='', - cert_file=None, auth_token=None): - """ - Instantiate Soledad. - """ - user = user or self.user - - # this callback ensures we save a document which is sent to the shared - # db. - def _put_doc_side_effect(doc): - self._doc_put = doc - - soledad = Soledad( - user, - passphrase, - secrets_path=os.path.join(self.tempdir, prefix, secrets_path), - local_db_path=os.path.join( - self.tempdir, prefix, local_db_path), - server_url=server_url, - cert_file=cert_file, - auth_token=auth_token, - shared_db=self.get_default_shared_mock(_put_doc_side_effect)) - self.addCleanup(soledad.close) - return soledad - - def make_app(self): - self.request_state = CouchServerState(self.couch_url) - return self.make_app_after_state(self.request_state) - - def setUp(self): - TestCaseWithServer.setUp(self) - CouchDBTestCase.setUp(self) - self.user = ('user-%s' % uuid4().hex) - self.db = CouchDatabase.open_database( - urljoin(self.couch_url, 'user-' + self.user), - create=True, - replica_uid='replica', - ensure_ddocs=True) - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.startTwistedServer() - - def tearDown(self): - self.db.delete_database() - self.db.close() - CouchDBTestCase.tearDown(self) - TestCaseWithServer.tearDown(self) - - # - # Sequential tests - # - - def test_correct_transaction_log_after_sequential_puts(self): - """ - Assert that the transaction_log increases accordingly with sequential - puts. - """ - doc = self.db.create_doc({'ops': 0}) - docs = [doc.doc_id] - for i in range(0, REPEAT_TIMES): - self.assertEqual( - i + 1, len(self.db._get_transaction_log())) - doc.content['ops'] += 1 - self.db.put_doc(doc) - docs.append(doc.doc_id) - - # assert length of transaction_log - transaction_log = self.db._get_transaction_log() - self.assertEqual( - REPEAT_TIMES + 1, len(transaction_log)) - - # assert that all entries in the log belong to the same doc - self.assertEqual(REPEAT_TIMES + 1, len(docs)) - for doc_id in docs: - self.assertEqual( - REPEAT_TIMES + 1, - len(filter(lambda t: t[0] == doc_id, transaction_log))) - - def test_correct_transaction_log_after_sequential_deletes(self): - """ - Assert that the transaction_log increases accordingly with sequential - puts and deletes. - """ - docs = [] - for i in range(0, REPEAT_TIMES): - doc = self.db.create_doc({'ops': 0}) - self.assertEqual( - 2 * i + 1, len(self.db._get_transaction_log())) - docs.append(doc.doc_id) - self.db.delete_doc(doc) - self.assertEqual( - 2 * i + 2, len(self.db._get_transaction_log())) - - # assert length of transaction_log - transaction_log = self.db._get_transaction_log() - self.assertEqual( - 2 * REPEAT_TIMES, len(transaction_log)) - - # assert that each doc appears twice in the transaction_log - self.assertEqual(REPEAT_TIMES, len(docs)) - for doc_id in docs: - self.assertEqual( - 2, - len(filter(lambda t: t[0] == doc_id, transaction_log))) - - @defer.inlineCallbacks - def test_correct_sync_log_after_sequential_syncs(self): - """ - Assert that the sync_log increases accordingly with sequential syncs. - """ - sol = self._soledad_instance( - auth_token='auth-token', - server_url=self.getURL()) - source_replica_uid = sol._dbpool.replica_uid - - def _create_docs(): - deferreds = [] - for i in xrange(0, REPEAT_TIMES): - deferreds.append(sol.create_doc({})) - return defer.gatherResults(deferreds) - - def _assert_transaction_and_sync_logs(results, sync_idx): - # assert sizes of transaction and sync logs - self.assertEqual( - sync_idx * REPEAT_TIMES, - len(self.db._get_transaction_log())) - gen, _ = self.db._get_replica_gen_and_trans_id(source_replica_uid) - self.assertEqual(sync_idx * REPEAT_TIMES, gen) - - def _assert_sync(results, sync_idx): - gen, docs = results - self.assertEqual((sync_idx + 1) * REPEAT_TIMES, gen) - self.assertEqual((sync_idx + 1) * REPEAT_TIMES, len(docs)) - # assert sizes of transaction and sync logs - self.assertEqual((sync_idx + 1) * REPEAT_TIMES, - len(self.db._get_transaction_log())) - target_known_gen, target_known_trans_id = \ - self.db._get_replica_gen_and_trans_id(source_replica_uid) - # assert it has the correct gen and trans_id - conn_key = sol._dbpool._u1dbconnections.keys().pop() - conn = sol._dbpool._u1dbconnections[conn_key] - sol_gen, sol_trans_id = conn._get_generation_info() - self.assertEqual(sol_gen, target_known_gen) - self.assertEqual(sol_trans_id, target_known_trans_id) - - # sync first time and assert success - results = yield _create_docs() - _assert_transaction_and_sync_logs(results, 0) - yield sol.sync() - results = yield sol.get_all_docs() - _assert_sync(results, 0) - - # create more docs, sync second time and assert success - results = yield _create_docs() - _assert_transaction_and_sync_logs(results, 1) - yield sol.sync() - results = yield sol.get_all_docs() - _assert_sync(results, 1) - - # - # Concurrency tests - # - - class _WorkerThread(threading.Thread): - - def __init__(self, params, run_method): - threading.Thread.__init__(self) - self._params = params - self._run_method = run_method - - def run(self): - self._run_method(self) - - def test_correct_transaction_log_after_concurrent_puts(self): - """ - Assert that the transaction_log increases accordingly with concurrent - puts. - """ - pool = threading.BoundedSemaphore(value=1) - threads = [] - docs = [] - - def _run_method(self): - doc = self._params['db'].create_doc({}) - pool.acquire() - self._params['docs'].append(doc.doc_id) - pool.release() - - for i in range(0, REPEAT_TIMES): - thread = self._WorkerThread( - {'docs': docs, 'db': self.db}, - _run_method) - thread.start() - threads.append(thread) - - for thread in threads: - thread.join() - - # assert length of transaction_log - transaction_log = self.db._get_transaction_log() - self.assertEqual( - REPEAT_TIMES, len(transaction_log)) - - # assert all documents are in the log - self.assertEqual(REPEAT_TIMES, len(docs)) - for doc_id in docs: - self.assertEqual( - 1, - len(filter(lambda t: t[0] == doc_id, transaction_log))) - - def test_correct_transaction_log_after_concurrent_deletes(self): - """ - Assert that the transaction_log increases accordingly with concurrent - puts and deletes. - """ - threads = [] - docs = [] - pool = threading.BoundedSemaphore(value=1) - - # create/delete method that will be run concurrently - def _run_method(self): - doc = self._params['db'].create_doc({}) - pool.acquire() - docs.append(doc.doc_id) - pool.release() - self._params['db'].delete_doc(doc) - - # launch concurrent threads - for i in range(0, REPEAT_TIMES): - thread = self._WorkerThread({'db': self.db}, _run_method) - thread.start() - threads.append(thread) - - # wait for threads to finish - for thread in threads: - thread.join() - - # assert transaction log - transaction_log = self.db._get_transaction_log() - self.assertEqual( - 2 * REPEAT_TIMES, len(transaction_log)) - # assert that each doc appears twice in the transaction_log - self.assertEqual(REPEAT_TIMES, len(docs)) - for doc_id in docs: - self.assertEqual( - 2, - len(filter(lambda t: t[0] == doc_id, transaction_log))) - - def test_correct_sync_log_after_concurrent_puts_and_sync(self): - """ - Assert that the sync_log is correct after concurrent syncs. - """ - docs = [] - - sol = self._soledad_instance( - auth_token='auth-token', - server_url=self.getURL()) - - def _save_doc_ids(results): - for doc in results: - docs.append(doc.doc_id) - - # create documents in parallel - deferreds = [] - for i in range(0, REPEAT_TIMES): - d = sol.create_doc({}) - deferreds.append(d) - - # wait for documents creation and sync - d = defer.gatherResults(deferreds) - d.addCallback(_save_doc_ids) - d.addCallback(lambda _: sol.sync()) - - def _assert_logs(results): - transaction_log = self.db._get_transaction_log() - self.assertEqual(REPEAT_TIMES, len(transaction_log)) - # assert all documents are in the remote log - self.assertEqual(REPEAT_TIMES, len(docs)) - for doc_id in docs: - self.assertEqual( - 1, - len(filter(lambda t: t[0] == doc_id, transaction_log))) - - d.addCallback(_assert_logs) - d.addCallback(lambda _: sol.close()) - - return d - - @defer.inlineCallbacks - def test_concurrent_syncs_do_not_fail(self): - """ - Assert that concurrent attempts to sync end up being executed - sequentially and do not fail. - """ - docs = [] - - sol = self._soledad_instance( - auth_token='auth-token', - server_url=self.getURL()) - - deferreds = [] - for i in xrange(0, REPEAT_TIMES): - d = sol.create_doc({}) - d.addCallback(lambda doc: docs.append(doc.doc_id)) - d.addCallback(lambda _: sol.sync()) - deferreds.append(d) - yield defer.gatherResults(deferreds, consumeErrors=True) - - transaction_log = self.db._get_transaction_log() - self.assertEqual(REPEAT_TIMES, len(transaction_log)) - # assert all documents are in the remote log - self.assertEqual(REPEAT_TIMES, len(docs)) - for doc_id in docs: - self.assertEqual( - 1, - len(filter(lambda t: t[0] == doc_id, transaction_log))) diff --git a/testing/tests/couch/test_ddocs.py b/testing/tests/couch/test_ddocs.py new file mode 100644 index 00000000..9ff32633 --- /dev/null +++ b/testing/tests/couch/test_ddocs.py @@ -0,0 +1,209 @@ +from uuid import uuid4 + +from leap.soledad.common.couch import errors +from leap.soledad.common import couch + +from test_soledad.util import CouchDBTestCase + + +class CouchDesignDocsTests(CouchDBTestCase): + + def setUp(self): + CouchDBTestCase.setUp(self) + + def create_db(self, ensure=True, dbname=None): + if not dbname: + dbname = ('test-%s' % uuid4().hex) + if dbname not in self.couch_server: + self.couch_server.create(dbname) + self.db = couch.CouchDatabase( + ('http://127.0.0.1:%d' % self.couch_port), + dbname, + ensure_ddocs=ensure) + + def tearDown(self): + self.db.delete_database() + self.db.close() + CouchDBTestCase.tearDown(self) + + def test_missing_design_doc_raises(self): + """ + Test that all methods that access design documents will raise if the + design docs are not present. + """ + self.create_db(ensure=False) + # get_generation_info() + self.assertRaises( + errors.MissingDesignDocError, + self.db.get_generation_info) + # get_trans_id_for_gen() + self.assertRaises( + errors.MissingDesignDocError, + self.db.get_trans_id_for_gen, 1) + # get_transaction_log() + self.assertRaises( + errors.MissingDesignDocError, + self.db.get_transaction_log) + # whats_changed() + self.assertRaises( + errors.MissingDesignDocError, + self.db.whats_changed) + + def test_missing_design_doc_functions_raises(self): + """ + Test that all methods that access design documents list functions + will raise if the functions are not present. + """ + self.create_db(ensure=True) + # erase views from _design/transactions + transactions = self.db._database['_design/transactions'] + transactions['lists'] = {} + self.db._database.save(transactions) + # get_generation_info() + self.assertRaises( + errors.MissingDesignDocListFunctionError, + self.db.get_generation_info) + # get_trans_id_for_gen() + self.assertRaises( + errors.MissingDesignDocListFunctionError, + self.db.get_trans_id_for_gen, 1) + # whats_changed() + self.assertRaises( + errors.MissingDesignDocListFunctionError, + self.db.whats_changed) + + def test_absent_design_doc_functions_raises(self): + """ + Test that all methods that access design documents list functions + will raise if the functions are not present. + """ + self.create_db(ensure=True) + # erase views from _design/transactions + transactions = self.db._database['_design/transactions'] + del transactions['lists'] + self.db._database.save(transactions) + # get_generation_info() + self.assertRaises( + errors.MissingDesignDocListFunctionError, + self.db.get_generation_info) + # _get_trans_id_for_gen() + self.assertRaises( + errors.MissingDesignDocListFunctionError, + self.db.get_trans_id_for_gen, 1) + # whats_changed() + self.assertRaises( + errors.MissingDesignDocListFunctionError, + self.db.whats_changed) + + def test_missing_design_doc_named_views_raises(self): + """ + Test that all methods that access design documents' named views will + raise if the views are not present. + """ + self.create_db(ensure=True) + # erase views from _design/docs + docs = self.db._database['_design/docs'] + del docs['views'] + self.db._database.save(docs) + # erase views from _design/syncs + syncs = self.db._database['_design/syncs'] + del syncs['views'] + self.db._database.save(syncs) + # erase views from _design/transactions + transactions = self.db._database['_design/transactions'] + del transactions['views'] + self.db._database.save(transactions) + # get_generation_info() + self.assertRaises( + errors.MissingDesignDocNamedViewError, + self.db.get_generation_info) + # _get_trans_id_for_gen() + self.assertRaises( + errors.MissingDesignDocNamedViewError, + self.db.get_trans_id_for_gen, 1) + # _get_transaction_log() + self.assertRaises( + errors.MissingDesignDocNamedViewError, + self.db.get_transaction_log) + # whats_changed() + self.assertRaises( + errors.MissingDesignDocNamedViewError, + self.db.whats_changed) + + def test_deleted_design_doc_raises(self): + """ + Test that all methods that access design documents will raise if the + design docs are not present. + """ + self.create_db(ensure=True) + # delete _design/docs + del self.db._database['_design/docs'] + # delete _design/syncs + del self.db._database['_design/syncs'] + # delete _design/transactions + del self.db._database['_design/transactions'] + # get_generation_info() + self.assertRaises( + errors.MissingDesignDocDeletedError, + self.db.get_generation_info) + # get_trans_id_for_gen() + self.assertRaises( + errors.MissingDesignDocDeletedError, + self.db.get_trans_id_for_gen, 1) + # get_transaction_log() + self.assertRaises( + errors.MissingDesignDocDeletedError, + self.db.get_transaction_log) + # whats_changed() + self.assertRaises( + errors.MissingDesignDocDeletedError, + self.db.whats_changed) + + def test_ensure_ddoc_independently(self): + """ + Test that a missing ddocs other than _design/docs will be ensured + even if _design/docs is there. + """ + self.create_db(ensure=True) + del self.db._database['_design/transactions'] + self.assertRaises( + errors.MissingDesignDocDeletedError, + self.db.get_transaction_log) + self.create_db(ensure=True, dbname=self.db._dbname) + self.db.get_transaction_log() + + def test_ensure_security_doc(self): + """ + Ensure_security creates a _security ddoc to ensure that only soledad + will have the lowest privileged access to an user db. + """ + self.create_db(ensure=False) + self.assertFalse(self.db._database.resource.get_json('_security')[2]) + self.db.ensure_security_ddoc() + security_ddoc = self.db._database.resource.get_json('_security')[2] + self.assertIn('admins', security_ddoc) + self.assertFalse(security_ddoc['admins']['names']) + self.assertIn('members', security_ddoc) + self.assertIn('soledad', security_ddoc['members']['names']) + + def test_ensure_security_from_configuration(self): + """ + Given a configuration, follow it to create the security document + """ + self.create_db(ensure=False) + configuration = {'members': ['user1', 'user2'], + 'members_roles': ['role1', 'role2'], + 'admins': ['admin'], + 'admins_roles': ['administrators'] + } + self.db.ensure_security_ddoc(configuration) + + security_ddoc = self.db._database.resource.get_json('_security')[2] + self.assertEquals(configuration['admins'], + security_ddoc['admins']['names']) + self.assertEquals(configuration['admins_roles'], + security_ddoc['admins']['roles']) + self.assertEquals(configuration['members'], + security_ddoc['members']['names']) + self.assertEquals(configuration['members_roles'], + security_ddoc['members']['roles']) diff --git a/testing/tests/couch/test_sync.py b/testing/tests/couch/test_sync.py new file mode 100644 index 00000000..bccbfe43 --- /dev/null +++ b/testing/tests/couch/test_sync.py @@ -0,0 +1,700 @@ +from leap.soledad.common.l2db import vectorclock +from leap.soledad.common.l2db import errors as u1db_errors + +from testscenarios import TestWithScenarios + +from test_soledad import u1db_tests as tests +from test_soledad.util import CouchDBTestCase +from test_soledad.util import sync_via_synchronizer +from test_soledad.u1db_tests import DatabaseBaseTests + +from common import simple_doc +from common import COUCH_SCENARIOS + + +sync_scenarios = [] +for name, scenario in COUCH_SCENARIOS: + scenario = dict(scenario) + scenario['do_sync'] = sync_via_synchronizer + sync_scenarios.append((name, scenario)) + scenario = dict(scenario) + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_sync`. +# ----------------------------------------------------------------------------- + +class CouchBackendSyncTests( + TestWithScenarios, + DatabaseBaseTests, + CouchDBTestCase): + + scenarios = sync_scenarios + + def create_database(self, replica_uid, sync_role=None): + if replica_uid == 'test' and sync_role is None: + # created up the chain by base class but unused + return None + db = self.create_database_for_role(replica_uid, sync_role) + if sync_role: + self._use_tracking[db] = (replica_uid, sync_role) + return db + + def create_database_for_role(self, replica_uid, sync_role): + # hook point for reuse + return DatabaseBaseTests.create_database(self, replica_uid) + + def copy_database(self, db, sync_role=None): + # DO NOT COPY OR REUSE THIS CODE OUTSIDE TESTS: COPYING U1DB DATABASES + # IS THE WRONG THING TO DO, THE ONLY REASON WE DO SO HERE IS TO TEST + # THAT WE CORRECTLY DETECT IT HAPPENING SO THAT WE CAN RAISE ERRORS + # RATHER THAN CORRUPT USER DATA. USE SYNC INSTEAD, OR WE WILL SEND + # NINJA TO YOUR HOUSE. + db_copy = self.copy_database_for_test(self, db) + name, orig_sync_role = self._use_tracking[db] + self._use_tracking[db_copy] = ( + name + '(copy)', sync_role or orig_sync_role) + return db_copy + + def sync(self, db_from, db_to, trace_hook=None, + trace_hook_shallow=None): + from_name, from_sync_role = self._use_tracking[db_from] + to_name, to_sync_role = self._use_tracking[db_to] + if from_sync_role not in ('source', 'both'): + raise Exception("%s marked for %s use but used as source" % + (from_name, from_sync_role)) + if to_sync_role not in ('target', 'both'): + raise Exception("%s marked for %s use but used as target" % + (to_name, to_sync_role)) + return self.do_sync(self, db_from, db_to, trace_hook, + trace_hook_shallow) + + def setUp(self): + self.db = None + self.db1 = None + self.db2 = None + self.db3 = None + self.db1_copy = None + self.db2_copy = None + self._use_tracking = {} + DatabaseBaseTests.setUp(self) + + def tearDown(self): + for db in [ + self.db, self.db1, self.db2, + self.db3, self.db1_copy, self.db2_copy + ]: + if db is not None: + self.delete_db(db._dbname) + db.close() + DatabaseBaseTests.tearDown(self) + + def assertLastExchangeLog(self, db, expected): + log = getattr(db, '_last_exchange_log', None) + if log is None: + return + self.assertEqual(expected, log) + + def test_sync_tracks_db_generation_of_other(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.assertEqual(0, self.sync(self.db1, self.db2)) + self.assertEqual( + (0, ''), self.db1._get_replica_gen_and_trans_id('test2')) + self.assertEqual( + (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [], 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 0}}) + + def test_sync_autoresolves(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(simple_doc, doc_id='doc') + rev1 = doc1.rev + doc2 = self.db2.create_doc_from_json(simple_doc, doc_id='doc') + rev2 = doc2.rev + self.sync(self.db1, self.db2) + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + self.assertEqual(doc.rev, self.db2.get_doc('doc').rev) + v = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev1))) + self.assertTrue(v.is_newer(vectorclock.VectorClockRev(rev2))) + + def test_sync_autoresolves_moar(self): + # here we test that when a database that has a conflicted document is + # the source of a sync, and the target database has a revision of the + # conflicted document that is newer than the source database's, and + # that target's database's document's content is the same as the + # source's document's conflict's, the source's document's conflict gets + # autoresolved, and the source's document's revision bumped. + # + # idea is as follows: + # A B + # a1 - + # `-------> + # a1 a1 + # v v + # a2 a1b1 + # `-------> + # a1b1+a2 a1b1 + # v + # a1b1+a2 a1b2 (a1b2 has same content as a2) + # `-------> + # a3b2 a1b2 (autoresolved) + # `-------> + # a3b2 a3b2 + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(simple_doc, doc_id='doc') + self.sync(self.db1, self.db2) + for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: + doc = db.get_doc('doc') + doc.set_json(content) + db.put_doc(doc) + self.sync(self.db1, self.db2) + # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict + doc = self.db1.get_doc('doc') + rev1 = doc.rev + self.assertTrue(doc.has_conflicts) + # set db2 to have a doc of {} (same as db1 before the conflict) + doc = self.db2.get_doc('doc') + doc.set_json('{}') + self.db2.put_doc(doc) + rev2 = doc.rev + # sync it across + self.sync(self.db1, self.db2) + # tadaa! + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + vec1 = vectorclock.VectorClockRev(rev1) + vec2 = vectorclock.VectorClockRev(rev2) + vec3 = vectorclock.VectorClockRev(doc.rev) + self.assertTrue(vec3.is_newer(vec1)) + self.assertTrue(vec3.is_newer(vec2)) + # because the conflict is on the source, sync it another time + self.sync(self.db1, self.db2) + # make sure db2 now has the exact same thing + self.assertEqual(self.db1.get_doc('doc'), self.db2.get_doc('doc')) + + def test_sync_autoresolves_moar_backwards(self): + # here we test that when a database that has a conflicted document is + # the target of a sync, and the source database has a revision of the + # conflicted document that is newer than the target database's, and + # that source's database's document's content is the same as the + # target's document's conflict's, the target's document's conflict gets + # autoresolved, and the document's revision bumped. + # + # idea is as follows: + # A B + # a1 - + # `-------> + # a1 a1 + # v v + # a2 a1b1 + # `-------> + # a1b1+a2 a1b1 + # v + # a1b1+a2 a1b2 (a1b2 has same content as a2) + # <-------' + # a3b2 a3b2 (autoresolved and propagated) + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'both') + self.db1.create_doc_from_json(simple_doc, doc_id='doc') + self.sync(self.db1, self.db2) + for db, content in [(self.db1, '{}'), (self.db2, '{"hi": 42}')]: + doc = db.get_doc('doc') + doc.set_json(content) + db.put_doc(doc) + self.sync(self.db1, self.db2) + # db1 and db2 now both have a doc of {hi:42}, but db1 has a conflict + doc = self.db1.get_doc('doc') + rev1 = doc.rev + self.assertTrue(doc.has_conflicts) + revc = self.db1.get_doc_conflicts('doc')[-1].rev + # set db2 to have a doc of {} (same as db1 before the conflict) + doc = self.db2.get_doc('doc') + doc.set_json('{}') + self.db2.put_doc(doc) + rev2 = doc.rev + # sync it across + self.sync(self.db2, self.db1) + # tadaa! + doc = self.db1.get_doc('doc') + self.assertFalse(doc.has_conflicts) + vec1 = vectorclock.VectorClockRev(rev1) + vec2 = vectorclock.VectorClockRev(rev2) + vec3 = vectorclock.VectorClockRev(doc.rev) + vecc = vectorclock.VectorClockRev(revc) + self.assertTrue(vec3.is_newer(vec1)) + self.assertTrue(vec3.is_newer(vec2)) + self.assertTrue(vec3.is_newer(vecc)) + # make sure db2 now has the exact same thing + self.assertEqual(self.db1.get_doc('doc'), self.db2.get_doc('doc')) + + def test_sync_autoresolves_moar_backwards_three(self): + # same as autoresolves_moar_backwards, but with three databases (note + # all the syncs go in the same direction -- this is a more natural + # scenario): + # + # A B C + # a1 - - + # `-------> + # a1 a1 - + # `-------> + # a1 a1 a1 + # v v + # a2 a1b1 a1 + # `-------------------> + # a2 a1b1 a2 + # `-------> + # a2+a1b1 a2 + # v + # a2 a2+a1b1 a2c1 (same as a1b1) + # `-------------------> + # a2c1 a2+a1b1 a2c1 + # `-------> + # a2b2c1 a2b2c1 a2c1 + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'both') + self.db3 = self.create_database('test3', 'target') + self.db1.create_doc_from_json(simple_doc, doc_id='doc') + self.sync(self.db1, self.db2) + self.sync(self.db2, self.db3) + for db, content in [(self.db2, '{"hi": 42}'), + (self.db1, '{}'), + ]: + doc = db.get_doc('doc') + doc.set_json(content) + db.put_doc(doc) + self.sync(self.db1, self.db3) + self.sync(self.db2, self.db3) + # db2 and db3 now both have a doc of {}, but db2 has a + # conflict + doc = self.db2.get_doc('doc') + self.assertTrue(doc.has_conflicts) + revc = self.db2.get_doc_conflicts('doc')[-1].rev + self.assertEqual('{}', doc.get_json()) + self.assertEqual(self.db3.get_doc('doc').get_json(), doc.get_json()) + self.assertEqual(self.db3.get_doc('doc').rev, doc.rev) + # set db3 to have a doc of {hi:42} (same as db2 before the conflict) + doc = self.db3.get_doc('doc') + doc.set_json('{"hi": 42}') + self.db3.put_doc(doc) + rev3 = doc.rev + # sync it across to db1 + self.sync(self.db1, self.db3) + # db1 now has hi:42, with a rev that is newer than db2's doc + doc = self.db1.get_doc('doc') + rev1 = doc.rev + self.assertFalse(doc.has_conflicts) + self.assertEqual('{"hi": 42}', doc.get_json()) + VCR = vectorclock.VectorClockRev + self.assertTrue(VCR(rev1).is_newer(VCR(self.db2.get_doc('doc').rev))) + # so sync it to db2 + self.sync(self.db1, self.db2) + # tadaa! + doc = self.db2.get_doc('doc') + self.assertFalse(doc.has_conflicts) + # db2's revision of the document is strictly newer than db1's before + # the sync, and db3's before that sync way back when + self.assertTrue(VCR(doc.rev).is_newer(VCR(rev1))) + self.assertTrue(VCR(doc.rev).is_newer(VCR(rev3))) + self.assertTrue(VCR(doc.rev).is_newer(VCR(revc))) + # make sure both dbs now have the exact same thing + self.assertEqual(self.db1.get_doc('doc'), self.db2.get_doc('doc')) + + def test_sync_puts_changes(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db1.create_doc_from_json(simple_doc) + self.assertEqual(1, self.sync(self.db1, self.db2)) + self.assertGetDoc(self.db2, doc.doc_id, doc.rev, simple_doc, False) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [(doc.doc_id, doc.rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 1}}) + + def test_sync_pulls_changes(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db2.create_doc_from_json(simple_doc) + self.assertEqual(0, self.sync(self.db1, self.db2)) + self.assertGetDoc(self.db1, doc.doc_id, doc.rev, simple_doc, False) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + self.assertEqual(1, self.db2._get_replica_gen_and_trans_id('test1')[0]) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [], 'last_known_gen': 0}, + 'return': {'docs': [(doc.doc_id, doc.rev)], + 'last_gen': 1}}) + self.assertGetDoc(self.db2, doc.doc_id, doc.rev, simple_doc, False) + + def test_sync_pulling_doesnt_update_other_if_changed(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db2.create_doc_from_json(simple_doc) + # After the local side has sent its list of docs, before we start + # receiving the "targets" response, we update the local database with a + # new record. + # When we finish synchronizing, we can notice that something locally + # was updated, and we cannot tell c2 our new updated generation + + def before_get_docs(state): + if state != 'before get_docs': + return + self.db1.create_doc_from_json(simple_doc) + + self.assertEqual(0, self.sync(self.db1, self.db2, + trace_hook=before_get_docs)) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [], 'last_known_gen': 0}, + 'return': {'docs': [(doc.doc_id, doc.rev)], + 'last_gen': 1}}) + self.assertEqual(1, self.db1._get_replica_gen_and_trans_id('test2')[0]) + # c2 should not have gotten a '_record_sync_info' call, because the + # local database had been updated more than just by the messages + # returned from c2. + self.assertEqual( + (0, ''), self.db2._get_replica_gen_and_trans_id('test1')) + + def test_sync_doesnt_update_other_if_nothing_pulled(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(simple_doc) + + def no_record_sync_info(state): + if state != 'record_sync_info': + return + self.fail('SyncTarget.record_sync_info was called') + self.assertEqual(1, self.sync(self.db1, self.db2, + trace_hook_shallow=no_record_sync_info)) + self.assertEqual( + 1, + self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)[0]) + + def test_sync_ignores_convergence(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'both') + doc = self.db1.create_doc_from_json(simple_doc) + self.db3 = self.create_database('test3', 'target') + self.assertEqual(1, self.sync(self.db1, self.db3)) + self.assertEqual(0, self.sync(self.db2, self.db3)) + self.assertEqual(1, self.sync(self.db1, self.db2)) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [(doc.doc_id, doc.rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 1}}) + + def test_sync_ignores_superseded(self): + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'both') + doc = self.db1.create_doc_from_json(simple_doc) + doc_rev1 = doc.rev + self.db3 = self.create_database('test3', 'target') + self.sync(self.db1, self.db3) + self.sync(self.db2, self.db3) + new_content = '{"key": "altval"}' + doc.set_json(new_content) + self.db1.put_doc(doc) + doc_rev2 = doc.rev + self.sync(self.db2, self.db1) + self.assertLastExchangeLog( + self.db1, + {'receive': {'docs': [(doc.doc_id, doc_rev1)], + 'source_uid': 'test2', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [(doc.doc_id, doc_rev2)], + 'last_gen': 2}}) + self.assertGetDoc(self.db1, doc.doc_id, doc_rev2, new_content, False) + + def test_sync_sees_remote_conflicted(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(simple_doc) + doc_id = doc1.doc_id + doc1_rev = doc1.rev + new_doc = '{"key": "altval"}' + doc2 = self.db2.create_doc_from_json(new_doc, doc_id=doc_id) + doc2_rev = doc2.rev + self.assertTransactionLog([doc1.doc_id], self.db1) + self.sync(self.db1, self.db2) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [(doc_id, doc1_rev)], + 'source_uid': 'test1', + 'source_gen': 1, 'last_known_gen': 0}, + 'return': {'docs': [(doc_id, doc2_rev)], + 'last_gen': 1}}) + self.assertTransactionLog([doc_id, doc_id], self.db1) + self.assertGetDoc(self.db1, doc_id, doc2_rev, new_doc, True) + self.assertGetDoc(self.db2, doc_id, doc2_rev, new_doc, False) + + def test_sync_sees_remote_delete_conflicted(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc1 = self.db1.create_doc_from_json(simple_doc) + doc_id = doc1.doc_id + self.sync(self.db1, self.db2) + doc2 = self.make_document(doc1.doc_id, doc1.rev, doc1.get_json()) + new_doc = '{"key": "altval"}' + doc1.set_json(new_doc) + self.db1.put_doc(doc1) + self.db2.delete_doc(doc2) + self.assertTransactionLog([doc_id, doc_id], self.db1) + self.sync(self.db1, self.db2) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [(doc_id, doc1.rev)], + 'source_uid': 'test1', + 'source_gen': 2, 'last_known_gen': 1}, + 'return': {'docs': [(doc_id, doc2.rev)], + 'last_gen': 2}}) + self.assertTransactionLog([doc_id, doc_id, doc_id], self.db1) + self.assertGetDocIncludeDeleted(self.db1, doc_id, doc2.rev, None, True) + self.assertGetDocIncludeDeleted( + self.db2, doc_id, doc2.rev, None, False) + + def test_sync_local_race_conflicted(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + doc = self.db1.create_doc_from_json(simple_doc) + doc_id = doc.doc_id + doc1_rev = doc.rev + self.sync(self.db1, self.db2) + content1 = '{"key": "localval"}' + content2 = '{"key": "altval"}' + doc.set_json(content2) + self.db2.put_doc(doc) + doc2_rev2 = doc.rev + triggered = [] + + def after_whatschanged(state): + if state != 'after whats_changed': + return + triggered.append(True) + doc = self.make_document(doc_id, doc1_rev, content1) + self.db1.put_doc(doc) + + self.sync(self.db1, self.db2, trace_hook=after_whatschanged) + self.assertEqual([True], triggered) + self.assertGetDoc(self.db1, doc_id, doc2_rev2, content2, True) + + def test_sync_propagates_deletes(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'both') + doc1 = self.db1.create_doc_from_json(simple_doc) + doc_id = doc1.doc_id + self.sync(self.db1, self.db2) + self.db3 = self.create_database('test3', 'target') + self.sync(self.db1, self.db3) + self.db1.delete_doc(doc1) + deleted_rev = doc1.rev + self.sync(self.db1, self.db2) + self.assertLastExchangeLog( + self.db2, + {'receive': {'docs': [(doc_id, deleted_rev)], + 'source_uid': 'test1', + 'source_gen': 2, 'last_known_gen': 1}, + 'return': {'docs': [], 'last_gen': 2}}) + self.assertGetDocIncludeDeleted( + self.db1, doc_id, deleted_rev, None, False) + self.assertGetDocIncludeDeleted( + self.db2, doc_id, deleted_rev, None, False) + self.sync(self.db2, self.db3) + self.assertLastExchangeLog( + self.db3, + {'receive': {'docs': [(doc_id, deleted_rev)], + 'source_uid': 'test2', + 'source_gen': 2, 'last_known_gen': 0}, + 'return': {'docs': [], 'last_gen': 2}}) + self.assertGetDocIncludeDeleted( + self.db3, doc_id, deleted_rev, None, False) + + def test_sync_propagates_deletes_2(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json('{"a": "1"}', doc_id='the-doc') + self.sync(self.db1, self.db2) + doc1_2 = self.db2.get_doc('the-doc') + self.db2.delete_doc(doc1_2) + self.sync(self.db1, self.db2) + self.assertGetDocIncludeDeleted( + self.db1, 'the-doc', doc1_2.rev, None, False) + + def test_sync_propagates_resolution(self): + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'both') + doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') + self.db3 = self.create_database('test3', 'both') + self.sync(self.db2, self.db1) + self.assertEqual( + self.db1._get_generation_info(), + self.db2._get_replica_gen_and_trans_id(self.db1._replica_uid)) + self.assertEqual( + self.db2._get_generation_info(), + self.db1._get_replica_gen_and_trans_id(self.db2._replica_uid)) + self.sync(self.db3, self.db1) + # update on 2 + doc2 = self.make_document('the-doc', doc1.rev, '{"a": 2}') + self.db2.put_doc(doc2) + self.sync(self.db2, self.db3) + self.assertEqual(self.db3.get_doc('the-doc').rev, doc2.rev) + # update on 1 + doc1.set_json('{"a": 3}') + self.db1.put_doc(doc1) + # conflicts + self.sync(self.db2, self.db1) + self.sync(self.db3, self.db1) + self.assertTrue(self.db2.get_doc('the-doc').has_conflicts) + self.assertTrue(self.db3.get_doc('the-doc').has_conflicts) + # resolve + conflicts = self.db2.get_doc_conflicts('the-doc') + doc4 = self.make_document('the-doc', None, '{"a": 4}') + revs = [doc.rev for doc in conflicts] + self.db2.resolve_doc(doc4, revs) + doc2 = self.db2.get_doc('the-doc') + self.assertEqual(doc4.get_json(), doc2.get_json()) + self.assertFalse(doc2.has_conflicts) + self.sync(self.db2, self.db3) + doc3 = self.db3.get_doc('the-doc') + self.assertEqual(doc4.get_json(), doc3.get_json()) + self.assertFalse(doc3.has_conflicts) + + def test_sync_supersedes_conflicts(self): + self.db1 = self.create_database('test1', 'both') + self.db2 = self.create_database('test2', 'target') + self.db3 = self.create_database('test3', 'both') + doc1 = self.db1.create_doc_from_json('{"a": 1}', doc_id='the-doc') + self.db2.create_doc_from_json('{"b": 1}', doc_id='the-doc') + self.db3.create_doc_from_json('{"c": 1}', doc_id='the-doc') + self.sync(self.db3, self.db1) + self.assertEqual( + self.db1._get_generation_info(), + self.db3._get_replica_gen_and_trans_id(self.db1._replica_uid)) + self.assertEqual( + self.db3._get_generation_info(), + self.db1._get_replica_gen_and_trans_id(self.db3._replica_uid)) + self.sync(self.db3, self.db2) + self.assertEqual( + self.db2._get_generation_info(), + self.db3._get_replica_gen_and_trans_id(self.db2._replica_uid)) + self.assertEqual( + self.db3._get_generation_info(), + self.db2._get_replica_gen_and_trans_id(self.db3._replica_uid)) + self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) + doc1.set_json('{"a": 2}') + self.db1.put_doc(doc1) + self.sync(self.db3, self.db1) + # original doc1 should have been removed from conflicts + self.assertEqual(3, len(self.db3.get_doc_conflicts('the-doc'))) + + def test_sync_stops_after_get_sync_info(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc) + self.sync(self.db1, self.db2) + + def put_hook(state): + self.fail("Tracehook triggered for %s" % (state,)) + + self.sync(self.db1, self.db2, trace_hook_shallow=put_hook) + + def test_sync_detects_identical_replica_uid(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test1', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') + self.assertRaises( + u1db_errors.InvalidReplicaUID, self.sync, self.db1, self.db2) + + def test_sync_detects_rollback_in_source(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') + self.sync(self.db1, self.db2) + self.db1_copy = self.copy_database(self.db1) + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.sync(self.db1, self.db2) + self.assertRaises( + u1db_errors.InvalidGeneration, self.sync, self.db1_copy, self.db2) + + def test_sync_detects_rollback_in_target(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.db2_copy = self.copy_database(self.db2) + self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.sync(self.db1, self.db2) + self.assertRaises( + u1db_errors.InvalidGeneration, self.sync, self.db1, self.db2_copy) + + def test_sync_detects_diverged_source(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db3 = self.copy_database(self.db1) + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.db3.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.assertRaises( + u1db_errors.InvalidTransactionId, self.sync, self.db3, self.db2) + + def test_sync_detects_diverged_target(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db3 = self.copy_database(self.db2) + self.db3.create_doc_from_json(tests.nested_doc, doc_id="divergent") + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.assertRaises( + u1db_errors.InvalidTransactionId, self.sync, self.db1, self.db3) + + def test_sync_detects_rollback_and_divergence_in_source(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc1') + self.sync(self.db1, self.db2) + self.db1_copy = self.copy_database(self.db1) + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db1.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.sync(self.db1, self.db2) + self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db1_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.assertRaises( + u1db_errors.InvalidTransactionId, self.sync, + self.db1_copy, self.db2) + + def test_sync_detects_rollback_and_divergence_in_target(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + self.db1.create_doc_from_json(tests.simple_doc, doc_id="divergent") + self.sync(self.db1, self.db2) + self.db2_copy = self.copy_database(self.db2) + self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db2.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.sync(self.db1, self.db2) + self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc2') + self.db2_copy.create_doc_from_json(tests.simple_doc, doc_id='doc3') + self.assertRaises( + u1db_errors.InvalidTransactionId, self.sync, + self.db1, self.db2_copy) + + def test_optional_sync_preserve_json(self): + self.db1 = self.create_database('test1', 'source') + self.db2 = self.create_database('test2', 'target') + cont1 = '{"a": 2}' + cont2 = '{"b": 3}' + self.db1.create_doc_from_json(cont1, doc_id="1") + self.db2.create_doc_from_json(cont2, doc_id="2") + self.sync(self.db1, self.db2) + self.assertEqual(cont1, self.db2.get_doc("1").get_json()) + self.assertEqual(cont2, self.db1.get_doc("2").get_json()) diff --git a/testing/tests/couch/test_sync_target.py b/testing/tests/couch/test_sync_target.py new file mode 100644 index 00000000..e792fb76 --- /dev/null +++ b/testing/tests/couch/test_sync_target.py @@ -0,0 +1,343 @@ +import json + +from leap.soledad.common.l2db import SyncTarget +from leap.soledad.common.l2db import errors as u1db_errors + +from testscenarios import TestWithScenarios + +from test_soledad import u1db_tests as tests +from test_soledad.util import CouchDBTestCase +from test_soledad.util import make_local_db_and_target +from test_soledad.u1db_tests import DatabaseBaseTests + +from common import simple_doc +from common import nested_doc +from common import COUCH_SCENARIOS + + +target_scenarios = [ + ('local', {'create_db_and_target': make_local_db_and_target}), ] + + +class CouchBackendSyncTargetTests( + TestWithScenarios, + DatabaseBaseTests, + CouchDBTestCase): + + # TODO: implement _set_trace_hook(_shallow) in CouchSyncTarget so + # skipped tests can be succesfully executed. + + # whitebox true means self.db is the actual local db object + # against which the sync is performed + whitebox = True + + scenarios = (tests.multiply_scenarios(COUCH_SCENARIOS, target_scenarios)) + + def set_trace_hook(self, callback, shallow=False): + setter = (self.st._set_trace_hook if not shallow else + self.st._set_trace_hook_shallow) + try: + setter(callback) + except NotImplementedError: + self.skipTest("%s does not implement _set_trace_hook" + % (self.st.__class__.__name__,)) + + def setUp(self): + CouchDBTestCase.setUp(self) + # other stuff + self.db, self.st = self.create_db_and_target(self) + self.other_changes = [] + + def tearDown(self): + self.db.close() + CouchDBTestCase.tearDown(self) + + def receive_doc(self, doc, gen, trans_id): + self.other_changes.append( + (doc.doc_id, doc.rev, doc.get_json(), gen, trans_id)) + + def test_sync_exchange_returns_many_new_docs(self): + # This test was replicated to allow dictionaries to be compared after + # JSON expansion (because one dictionary may have many different + # serialized representations). + doc = self.db.create_doc_from_json(simple_doc) + doc2 = self.db.create_doc_from_json(nested_doc) + self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) + new_gen, _ = self.st.sync_exchange( + [], 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id, doc2.doc_id], self.db) + self.assertEqual(2, new_gen) + self.assertEqual( + [(doc.doc_id, doc.rev, json.loads(simple_doc), 1), + (doc2.doc_id, doc2.rev, json.loads(nested_doc), 2)], + [c[:-3] + (json.loads(c[-3]), c[-2]) for c in self.other_changes]) + if self.whitebox: + self.assertEqual( + self.db._last_exchange_log['return'], + {'last_gen': 2, 'docs': + [(doc.doc_id, doc.rev), (doc2.doc_id, doc2.rev)]}) + + def test_get_sync_target(self): + self.assertIsNot(None, self.st) + + def test_get_sync_info(self): + self.assertEqual( + ('test', 0, '', 0, ''), self.st.get_sync_info('other')) + + def test_create_doc_updates_sync_info(self): + self.assertEqual( + ('test', 0, '', 0, ''), self.st.get_sync_info('other')) + self.db.create_doc_from_json(simple_doc) + self.assertEqual(1, self.st.get_sync_info('other')[1]) + + def test_record_sync_info(self): + self.st.record_sync_info('replica', 10, 'T-transid') + self.assertEqual( + ('test', 0, '', 10, 'T-transid'), self.st.get_sync_info('replica')) + + def test_sync_exchange(self): + docs_by_gen = [ + (self.make_document('doc-id', 'replica:1', simple_doc), 10, + 'T-sid')] + new_gen, trans_id = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertGetDoc(self.db, 'doc-id', 'replica:1', simple_doc, False) + self.assertTransactionLog(['doc-id'], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 1, last_trans_id), + (self.other_changes, new_gen, last_trans_id)) + self.assertEqual(10, self.st.get_sync_info('replica')[3]) + + def test_sync_exchange_deleted(self): + doc = self.db.create_doc_from_json('{}') + edit_rev = 'replica:1|' + doc.rev + docs_by_gen = [ + (self.make_document(doc.doc_id, edit_rev, None), 10, 'T-sid')] + new_gen, trans_id = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertGetDocIncludeDeleted( + self.db, doc.doc_id, edit_rev, None, False) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 2, last_trans_id), + (self.other_changes, new_gen, trans_id)) + self.assertEqual(10, self.st.get_sync_info('replica')[3]) + + def test_sync_exchange_push_many(self): + docs_by_gen = [ + (self.make_document('doc-id', 'replica:1', simple_doc), 10, 'T-1'), + (self.make_document('doc-id2', 'replica:1', nested_doc), 11, + 'T-2')] + new_gen, trans_id = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertGetDoc(self.db, 'doc-id', 'replica:1', simple_doc, False) + self.assertGetDoc(self.db, 'doc-id2', 'replica:1', nested_doc, False) + self.assertTransactionLog(['doc-id', 'doc-id2'], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 2, last_trans_id), + (self.other_changes, new_gen, trans_id)) + self.assertEqual(11, self.st.get_sync_info('replica')[3]) + + def test_sync_exchange_refuses_conflicts(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + (self.make_document(doc.doc_id, 'replica:1', new_doc), 10, + 'T-sid')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertEqual( + (doc.doc_id, doc.rev, simple_doc, 1), self.other_changes[0][:-1]) + self.assertEqual(1, new_gen) + if self.whitebox: + self.assertEqual(self.db._last_exchange_log['return'], + {'last_gen': 1, 'docs': [(doc.doc_id, doc.rev)]}) + + def test_sync_exchange_ignores_convergence(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + gen, txid = self.db._get_generation_info() + docs_by_gen = [ + (self.make_document(doc.doc_id, doc.rev, simple_doc), 10, 'T-sid')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'replica', last_known_generation=gen, + last_known_trans_id=txid, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertEqual(([], 1), (self.other_changes, new_gen)) + + def test_sync_exchange_returns_new_docs(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_gen, _ = self.st.sync_exchange( + [], 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertEqual( + (doc.doc_id, doc.rev, simple_doc, 1), self.other_changes[0][:-1]) + self.assertEqual(1, new_gen) + if self.whitebox: + self.assertEqual(self.db._last_exchange_log['return'], + {'last_gen': 1, 'docs': [(doc.doc_id, doc.rev)]}) + + def test_sync_exchange_returns_deleted_docs(self): + doc = self.db.create_doc_from_json(simple_doc) + self.db.delete_doc(doc) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + new_gen, _ = self.st.sync_exchange( + [], 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + self.assertEqual( + (doc.doc_id, doc.rev, None, 2), self.other_changes[0][:-1]) + self.assertEqual(2, new_gen) + if self.whitebox: + self.assertEqual(self.db._last_exchange_log['return'], + {'last_gen': 2, 'docs': [(doc.doc_id, doc.rev)]}) + + def test_sync_exchange_getting_newer_docs(self): + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + 'T-sid')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertTransactionLog([doc.doc_id, doc.doc_id], self.db) + self.assertEqual(([], 2), (self.other_changes, new_gen)) + + def test_sync_exchange_with_concurrent_updates_of_synced_doc(self): + expected = [] + + def before_whatschanged_cb(state): + if state != 'before whats_changed': + return + cont = '{"key": "cuncurrent"}' + conc_rev = self.db.put_doc( + self.make_document(doc.doc_id, 'test:1|z:2', cont)) + expected.append((doc.doc_id, conc_rev, cont, 3)) + + self.set_trace_hook(before_whatschanged_cb) + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + 'T-sid')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertEqual(expected, [c[:-1] for c in self.other_changes]) + self.assertEqual(3, new_gen) + + def test_sync_exchange_with_concurrent_updates(self): + + def after_whatschanged_cb(state): + if state != 'after whats_changed': + return + self.db.create_doc_from_json('{"new": "doc"}') + + self.set_trace_hook(after_whatschanged_cb) + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + new_doc = '{"key": "altval"}' + docs_by_gen = [ + (self.make_document(doc.doc_id, 'test:1|z:2', new_doc), 10, + 'T-sid')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertEqual(([], 2), (self.other_changes, new_gen)) + + def test_sync_exchange_converged_handling(self): + doc = self.db.create_doc_from_json(simple_doc) + docs_by_gen = [ + (self.make_document('new', 'other:1', '{}'), 4, 'T-foo'), + (self.make_document(doc.doc_id, doc.rev, doc.get_json()), 5, + 'T-bar')] + new_gen, _ = self.st.sync_exchange( + docs_by_gen, 'other-replica', last_known_generation=0, + last_known_trans_id=None, return_doc_cb=self.receive_doc) + self.assertEqual(([], 2), (self.other_changes, new_gen)) + + def test_sync_exchange_detect_incomplete_exchange(self): + def before_get_docs_explode(state): + if state != 'before get_docs': + return + raise u1db_errors.U1DBError("fail") + self.set_trace_hook(before_get_docs_explode) + # suppress traceback printing in the wsgiref server + # self.patch(simple_server.ServerHandler, + # 'log_exception', lambda h, exc_info: None) + doc = self.db.create_doc_from_json(simple_doc) + self.assertTransactionLog([doc.doc_id], self.db) + self.assertRaises( + (u1db_errors.U1DBError, u1db_errors.BrokenSyncStream), + self.st.sync_exchange, [], 'other-replica', + last_known_generation=0, last_known_trans_id=None, + return_doc_cb=self.receive_doc) + + def test_sync_exchange_doc_ids(self): + sync_exchange_doc_ids = getattr(self.st, 'sync_exchange_doc_ids', None) + if sync_exchange_doc_ids is None: + self.skipTest("sync_exchange_doc_ids not implemented") + db2 = self.create_database('test2') + doc = db2.create_doc_from_json(simple_doc) + new_gen, trans_id = sync_exchange_doc_ids( + db2, [(doc.doc_id, 10, 'T-sid')], 0, None, + return_doc_cb=self.receive_doc) + self.assertGetDoc(self.db, doc.doc_id, doc.rev, simple_doc, False) + self.assertTransactionLog([doc.doc_id], self.db) + last_trans_id = self.getLastTransId(self.db) + self.assertEqual(([], 1, last_trans_id), + (self.other_changes, new_gen, trans_id)) + self.assertEqual(10, self.st.get_sync_info(db2._replica_uid)[3]) + + def test__set_trace_hook(self): + called = [] + + def cb(state): + called.append(state) + + self.set_trace_hook(cb) + self.st.sync_exchange([], 'replica', 0, None, self.receive_doc) + self.st.record_sync_info('replica', 0, 'T-sid') + self.assertEqual(['before whats_changed', + 'after whats_changed', + 'before get_docs', + 'record_sync_info', + ], + called) + + def test__set_trace_hook_shallow(self): + st_trace_shallow = self.st._set_trace_hook_shallow + target_st_trace_shallow = SyncTarget._set_trace_hook_shallow + same_meth = st_trace_shallow == self.st._set_trace_hook + same_fun = st_trace_shallow.im_func == target_st_trace_shallow.im_func + if (same_meth or same_fun): + # shallow same as full + expected = ['before whats_changed', + 'after whats_changed', + 'before get_docs', + 'record_sync_info', + ] + else: + expected = ['sync_exchange', 'record_sync_info'] + + called = [] + + def cb(state): + called.append(state) + + self.set_trace_hook(cb, shallow=True) + self.st.sync_exchange([], 'replica', 0, None, self.receive_doc) + self.st.record_sync_info('replica', 0, 'T-sid') + self.assertEqual(expected, called) diff --git a/testing/tests/sqlcipher/hacker_crackdown.txt b/testing/tests/sqlcipher/hacker_crackdown.txt new file mode 100644 index 00000000..a01eb509 --- /dev/null +++ b/testing/tests/sqlcipher/hacker_crackdown.txt @@ -0,0 +1,13005 @@ +The Project Gutenberg EBook of Hacker Crackdown, by Bruce Sterling + +This eBook is for the use of anyone anywhere at no cost and with +almost no restrictions whatsoever. You may copy it, give it away or +re-use it under the terms of the Project Gutenberg License included +with this eBook or online at www.gutenberg.org + +** This is a COPYRIGHTED Project Gutenberg eBook, Details Below ** +** Please follow the copyright guidelines in this file. ** + +Title: Hacker Crackdown + Law and Disorder on the Electronic Frontier + +Author: Bruce Sterling + +Posting Date: February 9, 2012 [EBook #101] +Release Date: January, 1994 + +Language: English + + +*** START OF THIS PROJECT GUTENBERG EBOOK HACKER CRACKDOWN *** + + + + + + + + + + + + + +THE HACKER CRACKDOWN + +Law and Disorder on the Electronic Frontier + +by Bruce Sterling + + + + +CONTENTS + + +Preface to the Electronic Release of The Hacker Crackdown + +Chronology of the Hacker Crackdown + + +Introduction + + +Part 1: CRASHING THE SYSTEM +A Brief History of Telephony +Bell's Golden Vaporware +Universal Service +Wild Boys and Wire Women +The Electronic Communities +The Ungentle Giant +The Breakup +In Defense of the System +The Crash Post-Mortem +Landslides in Cyberspace + + +Part 2: THE DIGITAL UNDERGROUND +Steal This Phone +Phreaking and Hacking +The View From Under the Floorboards +Boards: Core of the Underground +Phile Phun +The Rake's Progress +Strongholds of the Elite +Sting Boards +Hot Potatoes +War on the Legion +Terminus +Phile 9-1-1 +War Games +Real Cyberpunk + + +Part 3: LAW AND ORDER +Crooked Boards +The World's Biggest Hacker Bust +Teach Them a Lesson +The U.S. Secret Service +The Secret Service Battles the Boodlers +A Walk Downtown +FCIC: The Cutting-Edge Mess +Cyberspace Rangers +FLETC: Training the Hacker-Trackers + + +Part 4: THE CIVIL LIBERTARIANS +NuPrometheus + FBI = Grateful Dead +Whole Earth + Computer Revolution = WELL +Phiber Runs Underground and Acid Spikes the Well +The Trial of Knight Lightning +Shadowhawk Plummets to Earth +Kyrie in the Confessional +$79,499 +A Scholar Investigates +Computers, Freedom, and Privacy + + +Electronic Afterword to The Hacker Crackdown, Halloween 1993 + + + + +THE HACKER CRACKDOWN + +Law and Disorder on the Electronic Frontier + +by Bruce Sterling + + + + + +Preface to the Electronic Release of The Hacker Crackdown + + +January 1, 1994--Austin, Texas + + +Hi, I'm Bruce Sterling, the author of this electronic book. + +Out in the traditional world of print, The Hacker Crackdown +is ISBN 0-553-08058-X, and is formally catalogued by +the Library of Congress as "1. Computer crimes--United States. +2. Telephone--United States--Corrupt practices. +3. Programming (Electronic computers)--United States--Corrupt practices." + +`Corrupt practices,' I always get a kick out of that description. +Librarians are very ingenious people. + +The paperback is ISBN 0-553-56370-X. If you go +and buy a print version of The Hacker Crackdown, +an action I encourage heartily, you may notice that +in the front of the book, beneath the copyright notice-- +"Copyright (C) 1992 by Bruce Sterling"-- +it has this little block of printed legal +boilerplate from the publisher. It says, and I quote: + + "No part of this book may be reproduced or transmitted in any form +or by any means, electronic or mechanical, including photocopying, +recording, or by any information storage and retrieval system, +without permission in writing from the publisher. +For information address: Bantam Books." + +This is a pretty good disclaimer, as such disclaimers go. +I collect intellectual-property disclaimers, and I've seen dozens of them, +and this one is at least pretty straightforward. In this narrow +and particular case, however, it isn't quite accurate. +Bantam Books puts that disclaimer on every book they publish, +but Bantam Books does not, in fact, own the electronic rights to this book. +I do, because of certain extensive contract maneuverings my agent and I +went through before this book was written. I want to give those electronic +publishing rights away through certain not-for-profit channels, +and I've convinced Bantam that this is a good idea. + +Since Bantam has seen fit to peacably agree to this scheme of mine, +Bantam Books is not going to fuss about this. Provided you don't try +to sell the book, they are not going to bother you for what you do with +the electronic copy of this book. If you want to check this out personally, +you can ask them; they're at 1540 Broadway NY NY 10036. However, if you were +so foolish as to print this book and start retailing it for money in violation +of my copyright and the commercial interests of Bantam Books, then Bantam, +a part of the gigantic Bertelsmann multinational publishing combine, +would roust some of their heavy-duty attorneys out of hibernation +and crush you like a bug. This is only to be expected. +I didn't write this book so that you could make money out of it. +If anybody is gonna make money out of this book, +it's gonna be me and my publisher. + +My publisher deserves to make money out of this book. +Not only did the folks at Bantam Books commission me +to write the book, and pay me a hefty sum to do so, but +they bravely printed, in text, an electronic document the +reproduction of which was once alleged to be a federal felony. +Bantam Books and their numerous attorneys were very brave +and forthright about this book. Furthermore, my former editor +at Bantam Books, Betsy Mitchell, genuinely cared about this project, +and worked hard on it, and had a lot of wise things to say +about the manuscript. Betsy deserves genuine credit for this book, +credit that editors too rarely get. + +The critics were very kind to The Hacker Crackdown, +and commercially the book has done well. On the other hand, +I didn't write this book in order to squeeze every last nickel +and dime out of the mitts of impoverished sixteen-year-old +cyberpunk high-school-students. Teenagers don't have any money-- +(no, not even enough for the six-dollar Hacker Crackdown paperback, +with its attractive bright-red cover and useful index). +That's a major reason why teenagers sometimes succumb to the temptation +to do things they shouldn't, such as swiping my books out of libraries. +Kids: this one is all yours, all right? Go give the print version back. +*8-) + +Well-meaning, public-spirited civil libertarians don't have much money, +either. And it seems almost criminal to snatch cash out of the hands of +America's direly underpaid electronic law enforcement community. + +If you're a computer cop, a hacker, or an electronic civil +liberties activist, you are the target audience for this book. +I wrote this book because I wanted to help you, and help other people +understand you and your unique, uhm, problems. I wrote this book +to aid your activities, and to contribute to the public discussion +of important political issues. In giving the text away in this +fashion, I am directly contributing to the book's ultimate aim: +to help civilize cyberspace. + +Information WANTS to be free. And the information inside +this book longs for freedom with a peculiar intensity. +I genuinely believe that the natural habitat of this book +is inside an electronic network. That may not be the easiest +direct method to generate revenue for the book's author, +but that doesn't matter; this is where this book belongs +by its nature. I've written other books--plenty of other books-- +and I'll write more and I am writing more, but this one is special. +I am making The Hacker Crackdown available electronically +as widely as I can conveniently manage, and if you like the book, +and think it is useful, then I urge you to do the same with it. + +You can copy this electronic book. Copy the heck out of it, +be my guest, and give those copies to anybody who wants them. +The nascent world of cyberspace is full of sysadmins, teachers, +trainers, cybrarians, netgurus, and various species of cybernetic activist. +If you're one of those people, I know about you, and I know the hassle +you go through to try to help people learn about the electronic frontier. +I hope that possessing this book in electronic form will lessen your troubles. +Granted, this treatment of our electronic social spectrum is not the ultimate +in academic rigor. And politically, it has something to offend +and trouble almost everyone. But hey, I'm told it's readable, +and at least the price is right. + +You can upload the book onto bulletin board systems, or Internet nodes, +or electronic discussion groups. Go right ahead and do that, I am giving +you express permission right now. Enjoy yourself. + +You can put the book on disks and give the disks away, +as long as you don't take any money for it. + +But this book is not public domain. You can't copyright it in +your own name. I own the copyright. Attempts to pirate this book +and make money from selling it may involve you in a serious litigative snarl. +Believe me, for the pittance you might wring out of such an action, +it's really not worth it. This book don't "belong" to you. +In an odd but very genuine way, I feel it doesn't "belong" to me, either. +It's a book about the people of cyberspace, and distributing it in this way +is the best way I know to actually make this information available, +freely and easily, to all the people of cyberspace--including people +far outside the borders of the United States, who otherwise may never +have a chance to see any edition of the book, and who may perhaps learn +something useful from this strange story of distant, obscure, but portentous +events in so-called "American cyberspace." + +This electronic book is now literary freeware. It now belongs to the +emergent realm of alternative information economics. You have no right +to make this electronic book part of the conventional flow of commerce. +Let it be part of the flow of knowledge: there's a difference. +I've divided the book into four sections, so that it is less ungainly +for upload and download; if there's a section of particular relevance +to you and your colleagues, feel free to reproduce that one and skip the rest. + +[Project Gutenberg has reassembled the file, with Sterling's permission.] + +Just make more when you need them, and give them to whoever might want them. + +Now have fun. + +Bruce Sterling--bruces@well.sf.ca.us + + +THE HACKER CRACKDOWN + +Law and Disorder on the Electronic Frontier + +by Bruce Sterling + + + + + + + +CHRONOLOGY OF THE HACKER CRACKDOWN + + +1865 U.S. Secret Service (USSS) founded. + +1876 Alexander Graham Bell invents telephone. + +1878 First teenage males flung off phone system by enraged authorities. + +1939 "Futurian" science-fiction group raided by Secret Service. + +1971 Yippie phone phreaks start YIPL/TAP magazine. + +1972 RAMPARTS magazine seized in blue-box rip-off scandal. + +1978 Ward Christenson and Randy Suess create first personal + computer bulletin board system. + +1982 William Gibson coins term "cyberspace." + +1982 "414 Gang" raided. + +1983-1983 AT&T dismantled in divestiture. + +1984 Congress passes Comprehensive Crime Control Act giving USSS + jurisdiction over credit card fraud and computer fraud. + +1984 "Legion of Doom" formed. + +1984. 2600: THE HACKER QUARTERLY founded. + +1984. WHOLE EARTH SOFTWARE CATALOG published. + +1985. First police "sting" bulletin board systems established. + +1985. Whole Earth 'Lectronic Link computer conference (WELL) goes on-line. + +1986 Computer Fraud and Abuse Act passed. + +1986 Electronic Communications Privacy Act passed. + +1987 Chicago prosecutors form Computer Fraud and Abuse Task Force. + + +1988 + +July. Secret Service covertly videotapes "SummerCon" hacker convention. + +September. "Prophet" cracks BellSouth AIMSX computer network + and downloads E911 Document to his own computer and to Jolnet. + +September. AT&T Corporate Information Security informed of Prophet's action. + +October. Bellcore Security informed of Prophet's action. + + +1989 + +January. Prophet uploads E911 Document to Knight Lightning. + +February 25. Knight Lightning publishes E911 Document in PHRACK + electronic newsletter. + +May. Chicago Task Force raids and arrests "Kyrie." + +June. "NuPrometheus League" distributes Apple Computer proprietary software. + +June 13. Florida probation office crossed with phone-sex line + in switching-station stunt. + +July. "Fry Guy" raided by USSS and Chicago Computer Fraud + and Abuse Task Force. + +July. Secret Service raids "Prophet," "Leftist," and "Urvile" in Georgia. + + +1990 + +January 15. Martin Luther King Day Crash strikes AT&T long-distance + network nationwide. + +January 18-19. Chicago Task Force raids Knight Lightning in St. Louis. + +January 24. USSS and New York State Police raid "Phiber Optik," + "Acid Phreak," and "Scorpion" in New York City. + +February 1. USSS raids "Terminus" in Maryland. + +February 3. Chicago Task Force raids Richard Andrews' home. + +February 6. Chicago Task Force raids Richard Andrews' business. + +February 6. USSS arrests Terminus, Prophet, Leftist, and Urvile. + +February 9. Chicago Task Force arrests Knight Lightning. + +February 20. AT&T Security shuts down public-access + "attctc" computer in Dallas. + +February 21. Chicago Task Force raids Robert Izenberg in Austin. + +March 1. Chicago Task Force raids Steve Jackson Games, Inc., + "Mentor," and "Erik Bloodaxe" in Austin. + +May 7,8,9. + +USSS and Arizona Organized Crime and Racketeering Bureau conduct +"Operation Sundevil" raids in Cincinnatti, Detroit, Los Angeles, +Miami, Newark, Phoenix, Pittsburgh, Richmond, Tucson, San Diego, +San Jose, and San Francisco. + +May. FBI interviews John Perry Barlow re NuPrometheus case. + +June. Mitch Kapor and Barlow found Electronic Frontier Foundation; + Barlow publishes CRIME AND PUZZLEMENT manifesto. + +July 24-27. Trial of Knight Lightning. + +1991 + +February. CPSR Roundtable in Washington, D.C. + +March 25-28. Computers, Freedom and Privacy conference in San Francisco. + +May 1. Electronic Frontier Foundation, Steve Jackson, + and others file suit against members of Chicago Task Force. + +July 1-2. Switching station phone software crash affects + Washington, Los Angeles, Pittsburgh, San Francisco. + +September 17. AT&T phone crash affects New York City and three airports. + + + + +Introduction + +This is a book about cops, and wild teenage whiz-kids, and lawyers, +and hairy-eyed anarchists, and industrial technicians, and hippies, +and high-tech millionaires, and game hobbyists, and computer security +experts, and Secret Service agents, and grifters, and thieves. + +This book is about the electronic frontier of the 1990s. +It concerns activities that take place inside computers +and over telephone lines. + +A science fiction writer coined the useful term "cyberspace" in 1982, +but the territory in question, the electronic frontier, is about +a hundred and thirty years old. Cyberspace is the "place" where +a telephone conversation appears to occur. Not inside your actual phone, +the plastic device on your desk. Not inside the other person's phone, +in some other city. THE PLACE BETWEEN the phones. The indefinite +place OUT THERE, where the two of you, two human beings, +actually meet and communicate. + +Although it is not exactly "real," "cyberspace" is a genuine place. +Things happen there that have very genuine consequences. This "place" +is not "real," but it is serious, it is earnest. Tens of thousands +of people have dedicated their lives to it, to the public service +of public communication by wire and electronics. + +People have worked on this "frontier" for generations now. +Some people became rich and famous from their efforts there. +Some just played in it, as hobbyists. Others soberly pondered it, +and wrote about it, and regulated it, and negotiated over it in +international forums, and sued one another about it, in gigantic, +epic court battles that lasted for years. And almost since +the beginning, some people have committed crimes in this place. + +But in the past twenty years, this electrical "space," +which was once thin and dark and one-dimensional--little more +than a narrow speaking-tube, stretching from phone to phone-- +has flung itself open like a gigantic jack-in-the-box. +Light has flooded upon it, the eerie light of the glowing computer screen. +This dark electric netherworld has become a vast flowering electronic landscape. +Since the 1960s, the world of the telephone has cross-bred itself +with computers and television, and though there is still no substance +to cyberspace, nothing you can handle, it has a strange kind +of physicality now. It makes good sense today to talk of cyberspace +as a place all its own. + +Because people live in it now. Not just a few people, +not just a few technicians and eccentrics, but thousands +of people, quite normal people. And not just for a little while, +either, but for hours straight, over weeks, and months, +and years. Cyberspace today is a "Net," a "Matrix," +international in scope and growing swiftly and steadily. +It's growing in size, and wealth, and political importance. + +People are making entire careers in modern cyberspace. +Scientists and technicians, of course; they've been there +for twenty years now. But increasingly, cyberspace +is filling with journalists and doctors and lawyers +and artists and clerks. Civil servants make their +careers there now, "on-line" in vast government data-banks; +and so do spies, industrial, political, and just plain snoops; +and so do police, at least a few of them. And there are children +living there now. + +People have met there and been married there. +There are entire living communities in cyberspace today; +chattering, gossiping, planning, conferring and scheming, +leaving one another voice-mail and electronic mail, +giving one another big weightless chunks of valuable data, +both legitimate and illegitimate. They busily pass one another +computer software and the occasional festering computer virus. + +We do not really understand how to live in cyberspace yet. +We are feeling our way into it, blundering about. +That is not surprising. Our lives in the physical world, +the "real" world, are also far from perfect, despite a lot more practice. +Human lives, real lives, are imperfect by their nature, and there are +human beings in cyberspace. The way we live in cyberspace is +a funhouse mirror of the way we live in the real world. +We take both our advantages and our troubles with us. + +This book is about trouble in cyberspace. +Specifically, this book is about certain strange events in +the year 1990, an unprecedented and startling year for the +the growing world of computerized communications. + +In 1990 there came a nationwide crackdown on illicit +computer hackers, with arrests, criminal charges, +one dramatic show-trial, several guilty pleas, and +huge confiscations of data and equipment all over the USA. + +The Hacker Crackdown of 1990 was larger, better organized, +more deliberate, and more resolute than any previous effort +in the brave new world of computer crime. The U.S. Secret Service, +private telephone security, and state and local law enforcement groups +across the country all joined forces in a determined attempt to break +the back of America's electronic underground. It was a fascinating +effort, with very mixed results. + +The Hacker Crackdown had another unprecedented effect; +it spurred the creation, within "the computer community," +of the Electronic Frontier Foundation, a new and very odd +interest group, fiercely dedicated to the establishment +and preservation of electronic civil liberties. The crackdown, +remarkable in itself, has created a melee of debate over electronic crime, +punishment, freedom of the press, and issues of search and seizure. +Politics has entered cyberspace. Where people go, politics follow. + +This is the story of the people of cyberspace. + + + +PART ONE: Crashing the System + +On January 15, 1990, AT&T's long-distance telephone switching system crashed. + +This was a strange, dire, huge event. Sixty thousand people lost +their telephone service completely. During the nine long hours +of frantic effort that it took to restore service, some seventy million +telephone calls went uncompleted. + +Losses of service, known as "outages" in the telco trade, +are a known and accepted hazard of the telephone business. +Hurricanes hit, and phone cables get snapped by the thousands. +Earthquakes wrench through buried fiber-optic lines. +Switching stations catch fire and burn to the ground. +These things do happen. There are contingency plans for them, +and decades of experience in dealing with them. +But the Crash of January 15 was unprecedented. +It was unbelievably huge, and it occurred for +no apparent physical reason. + +The crash started on a Monday afternoon in a single +switching-station in Manhattan. But, unlike any merely +physical damage, it spread and spread. Station after +station across America collapsed in a chain reaction, +until fully half of AT&T's network had gone haywire +and the remaining half was hard-put to handle the overflow. + +Within nine hours, AT&T software engineers more or less +understood what had caused the crash. Replicating the +problem exactly, poring over software line by line, +took them a couple of weeks. But because it was hard +to understand technically, the full truth of the matter +and its implications were not widely and thoroughly aired +and explained. The root cause of the crash remained obscure, +surrounded by rumor and fear. + +The crash was a grave corporate embarrassment. +The "culprit" was a bug in AT&T's own software--not the +sort of admission the telecommunications giant wanted +to make, especially in the face of increasing competition. +Still, the truth WAS told, in the baffling technical terms +necessary to explain it. + +Somehow the explanation failed to persuade +American law enforcement officials and even telephone +corporate security personnel. These people were not +technical experts or software wizards, and they had their +own suspicions about the cause of this disaster. + +The police and telco security had important sources +of information denied to mere software engineers. +They had informants in the computer underground and +years of experience in dealing with high-tech rascality +that seemed to grow ever more sophisticated. +For years they had been expecting a direct and +savage attack against the American national telephone system. +And with the Crash of January 15--the first month of a +new, high-tech decade--their predictions, fears, +and suspicions seemed at last to have entered the real world. +A world where the telephone system had not merely crashed, +but, quite likely, BEEN crashed--by "hackers." + +The crash created a large dark cloud of suspicion +that would color certain people's assumptions and actions +for months. The fact that it took place in the realm of +software was suspicious on its face. The fact that it +occurred on Martin Luther King Day, still the most +politically touchy of American holidays, made it more +suspicious yet. + +The Crash of January 15 gave the Hacker Crackdown +its sense of edge and its sweaty urgency. It made people, +powerful people in positions of public authority, +willing to believe the worst. And, most fatally, +it helped to give investigators a willingness +to take extreme measures and the determination +to preserve almost total secrecy. + +An obscure software fault in an aging switching system +in New York was to lead to a chain reaction of legal +and constitutional trouble all across the country. + +# + +Like the crash in the telephone system, this chain reaction +was ready and waiting to happen. During the 1980s, +the American legal system was extensively patched +to deal with the novel issues of computer crime. +There was, for instance, the Electronic Communications +Privacy Act of 1986 (eloquently described as "a stinking mess" +by a prominent law enforcement official). And there was the +draconian Computer Fraud and Abuse Act of 1986, passed unanimously +by the United States Senate, which later would reveal +a large number of flaws. Extensive, well-meant efforts +had been made to keep the legal system up to date. +But in the day-to-day grind of the real world, +even the most elegant software tends to crumble +and suddenly reveal its hidden bugs. + +Like the advancing telephone system, the American legal system +was certainly not ruined by its temporary crash; but for those +caught under the weight of the collapsing system, life became +a series of blackouts and anomalies. + +In order to understand why these weird events occurred, +both in the world of technology and in the world of law, +it's not enough to understand the merely technical problems. +We will get to those; but first and foremost, we must try +to understand the telephone, and the business of telephones, +and the community of human beings that telephones have created. + +# + +Technologies have life cycles, like cities do, +like institutions do, like laws and governments do. + +The first stage of any technology is the Question +Mark, often known as the "Golden Vaporware" stage. +At this early point, the technology is only a phantom, +a mere gleam in the inventor's eye. One such inventor +was a speech teacher and electrical tinkerer named +Alexander Graham Bell. + +Bell's early inventions, while ingenious, failed to move the world. +In 1863, the teenage Bell and his brother Melville made an artificial +talking mechanism out of wood, rubber, gutta-percha, and tin. +This weird device had a rubber-covered "tongue" made of movable +wooden segments, with vibrating rubber "vocal cords," and +rubber "lips" and "cheeks." While Melville puffed a bellows +into a tin tube, imitating the lungs, young Alec Bell would +manipulate the "lips," "teeth," and "tongue," causing the thing +to emit high-pitched falsetto gibberish. + +Another would-be technical breakthrough was the Bell "phonautograph" +of 1874, actually made out of a human cadaver's ear. Clamped into place +on a tripod, this grisly gadget drew sound-wave images on smoked glass +through a thin straw glued to its vibrating earbones. + +By 1875, Bell had learned to produce audible sounds--ugly shrieks +and squawks--by using magnets, diaphragms, and electrical current. + +Most "Golden Vaporware" technologies go nowhere. + +But the second stage of technology is the Rising Star, +or, the "Goofy Prototype," stage. The telephone, Bell's +most ambitious gadget yet, reached this stage on March +10, 1876. On that great day, Alexander Graham Bell +became the first person to transmit intelligible human +speech electrically. As it happened, young Professor Bell, +industriously tinkering in his Boston lab, had spattered +his trousers with acid. His assistant, Mr. Watson, +heard his cry for help--over Bell's experimental +audio-telegraph. This was an event without precedent. + +Technologies in their "Goofy Prototype" stage rarely +work very well. They're experimental, and therefore +half- baked and rather frazzled. The prototype may +be attractive and novel, and it does look as if it ought +to be good for something-or-other. But nobody, including +the inventor, is quite sure what. Inventors, and speculators, +and pundits may have very firm ideas about its potential +use, but those ideas are often very wrong. + +The natural habitat of the Goofy Prototype is in trade shows +and in the popular press. Infant technologies need publicity +and investment money like a tottering calf need milk. +This was very true of Bell's machine. To raise research and +development money, Bell toured with his device as a stage attraction. + +Contemporary press reports of the stage debut of the telephone +showed pleased astonishment mixed with considerable dread. +Bell's stage telephone was a large wooden box with a crude +speaker-nozzle, the whole contraption about the size and shape +of an overgrown Brownie camera. Its buzzing steel soundplate, +pumped up by powerful electromagnets, was loud enough to fill +an auditorium. Bell's assistant Mr. Watson, who could manage +on the keyboards fairly well, kicked in by playing the organ +from distant rooms, and, later, distant cities. This feat was +considered marvellous, but very eerie indeed. + +Bell's original notion for the telephone, an idea promoted +for a couple of years, was that it would become a mass medium. +We might recognize Bell's idea today as something close to modern +"cable radio." Telephones at a central source would transmit music, +Sunday sermons, and important public speeches to a paying network +of wired-up subscribers. + +At the time, most people thought this notion made good sense. +In fact, Bell's idea was workable. In Hungary, this philosophy +of the telephone was successfully put into everyday practice. +In Budapest, for decades, from 1893 until after World War I, +there was a government-run information service called +"Telefon Hirmondo-." Hirmondo- was a centralized source +of news and entertainment and culture, including stock reports, +plays, concerts, and novels read aloud. At certain hours +of the day, the phone would ring, you would plug in +a loudspeaker for the use of the family, and Telefon +Hirmondo- would be on the air--or rather, on the phone. + +Hirmondo- is dead tech today, but Hirmondo- might be considered +a spiritual ancestor of the modern telephone-accessed computer +data services, such as CompuServe, GEnie or Prodigy. +The principle behind Hirmondo- is also not too far from computer +"bulletin- board systems" or BBS's, which arrived in the late 1970s, +spread rapidly across America, and will figure largely in this book. + +We are used to using telephones for individual person-to-person speech, +because we are used to the Bell system. But this was just one possibility +among many. Communication networks are very flexible and protean, +especially when their hardware becomes sufficiently advanced. +They can be put to all kinds of uses. And they have been-- +and they will be. + +Bell's telephone was bound for glory, but this was a combination +of political decisions, canny infighting in court, inspired industrial +leadership, receptive local conditions and outright good luck. +Much the same is true of communications systems today. + +As Bell and his backers struggled to install their newfangled system +in the real world of nineteenth-century New England, they had to fight +against skepticism and industrial rivalry. There was already a strong +electrical communications network present in America: the telegraph. +The head of the Western Union telegraph system dismissed Bell's prototype +as "an electrical toy" and refused to buy the rights to Bell's patent. +The telephone, it seemed, might be all right as a parlor entertainment-- +but not for serious business. + +Telegrams, unlike mere telephones, left a permanent physical record +of their messages. Telegrams, unlike telephones, could be answered +whenever the recipient had time and convenience. And the telegram +had a much longer distance-range than Bell's early telephone. +These factors made telegraphy seem a much more sound and businesslike +technology--at least to some. + +The telegraph system was huge, and well-entrenched. +In 1876, the United States had 214,000 miles of telegraph wire, +and 8500 telegraph offices. There were specialized telegraphs +for businesses and stock traders, government, police and fire departments. +And Bell's "toy" was best known as a stage-magic musical device. + +The third stage of technology is known as the "Cash Cow" stage. +In the "cash cow" stage, a technology finds its place in the world, +and matures, and becomes settled and productive. After a year or so, +Alexander Graham Bell and his capitalist backers concluded that +eerie music piped from nineteenth-century cyberspace was not the real +selling-point of his invention. Instead, the telephone was about speech-- +individual, personal speech, the human voice, human conversation and +human interaction. The telephone was not to be managed from any centralized +broadcast center. It was to be a personal, intimate technology. + +When you picked up a telephone, you were not absorbing +the cold output of a machine--you were speaking to another human being. +Once people realized this, their instinctive dread of the telephone +as an eerie, unnatural device, swiftly vanished. A "telephone call" +was not a "call" from a "telephone" itself, but a call from another +human being, someone you would generally know and recognize. +The real point was not what the machine could do for you (or to you), +but what you yourself, a person and citizen, could do THROUGH the machine. +This decision on the part of the young Bell Company was absolutely vital. + +The first telephone networks went up around Boston--mostly among +the technically curious and the well-to-do (much the same segment +of the American populace that, a hundred years later, would be +buying personal computers). Entrenched backers of the telegraph +continued to scoff. + +But in January 1878, a disaster made the telephone famous. +A train crashed in Tarriffville, Connecticut. Forward-looking +doctors in the nearby city of Hartford had had Bell's +"speaking telephone" installed. An alert local druggist +was able to telephone an entire community of local doctors, +who rushed to the site to give aid. The disaster, as disasters do, +aroused intense press coverage. The phone had proven its usefulness +in the real world. + +After Tarriffville, the telephone network spread like crabgrass. +By 1890 it was all over New England. By '93, out to Chicago. +By '97, into Minnesota, Nebraska and Texas. By 1904 it was +all over the continent. + +The telephone had become a mature technology. Professor Bell +(now generally known as "Dr. Bell" despite his lack of a formal degree) +became quite wealthy. He lost interest in the tedious day-to-day business +muddle of the booming telephone network, and gratefully returned +his attention to creatively hacking-around in his various laboratories, +which were now much larger, better-ventilated, and gratifyingly +better-equipped. Bell was never to have another great inventive success, +though his speculations and prototypes anticipated fiber-optic transmission, +manned flight, sonar, hydrofoil ships, tetrahedral construction, and +Montessori education. The "decibel," the standard scientific measure +of sound intensity, was named after Bell. + +Not all Bell's vaporware notions were inspired. He was fascinated +by human eugenics. He also spent many years developing a weird personal +system of astrophysics in which gravity did not exist. + +Bell was a definite eccentric. He was something of a hypochondriac, +and throughout his life he habitually stayed up until four A.M., +refusing to rise before noon. But Bell had accomplished a great feat; +he was an idol of millions and his influence, wealth, and great +personal charm, combined with his eccentricity, made him something +of a loose cannon on deck. Bell maintained a thriving scientific +salon in his winter mansion in Washington, D.C., which gave him +considerable backstage influence in governmental and scientific circles. +He was a major financial backer of the the magazines Science and +National Geographic, both still flourishing today as important organs +of the American scientific establishment. + +Bell's companion Thomas Watson, similarly wealthy and similarly odd, +became the ardent political disciple of a 19th-century science-fiction writer +and would-be social reformer, Edward Bellamy. Watson also trod the boards +briefly as a Shakespearian actor. + +There would never be another Alexander Graham Bell, +but in years to come there would be surprising numbers +of people like him. Bell was a prototype of the +high-tech entrepreneur. High-tech entrepreneurs will +play a very prominent role in this book: not merely as +technicians and businessmen, but as pioneers of the +technical frontier, who can carry the power and prestige +they derive from high-technology into the political and +social arena. + +Like later entrepreneurs, Bell was fierce in defense of +his own technological territory. As the telephone began to +flourish, Bell was soon involved in violent lawsuits in the +defense of his patents. Bell's Boston lawyers were +excellent, however, and Bell himself, as an elocution +teacher and gifted public speaker, was a devastatingly +effective legal witness. In the eighteen years of Bell's patents, +the Bell company was involved in six hundred separate lawsuits. +The legal records printed filled 149 volumes. The Bell Company +won every single suit. + +After Bell's exclusive patents expired, rival telephone +companies sprang up all over America. Bell's company, +American Bell Telephone, was soon in deep trouble. +In 1907, American Bell Telephone fell into the hands of the +rather sinister J.P. Morgan financial cartel, robber-baron +speculators who dominated Wall Street. + +At this point, history might have taken a different turn. +American might well have been served forever by a patchwork +of locally owned telephone companies. Many state politicians +and local businessmen considered this an excellent solution. + +But the new Bell holding company, American Telephone and Telegraph +or AT&T, put in a new man at the helm, a visionary industrialist +named Theodore Vail. Vail, a former Post Office manager, +understood large organizations and had an innate feeling +for the nature of large-scale communications. Vail quickly +saw to it that AT&T seized the technological edge once again. +The Pupin and Campbell "loading coil," and the deForest +"audion," are both extinct technology today, but in 1913 +they gave Vail's company the best LONG-DISTANCE lines +ever built. By controlling long-distance--the links +between, and over, and above the smaller local phone +companies--AT&T swiftly gained the whip-hand over them, +and was soon devouring them right and left. + +Vail plowed the profits back into research and development, +starting the Bell tradition of huge-scale and brilliant +industrial research. + +Technically and financially, AT&T gradually steamrollered +the opposition. Independent telephone companies never +became entirely extinct, and hundreds of them flourish today. +But Vail's AT&T became the supreme communications company. +At one point, Vail's AT&T bought Western Union itself, +the very company that had derided Bell's telephone as a "toy." +Vail thoroughly reformed Western Union's hidebound business +along his modern principles; but when the federal government +grew anxious at this centralization of power, Vail politely +gave Western Union back. + +This centralizing process was not unique. Very similar +events had happened in American steel, oil, and railroads. +But AT&T, unlike the other companies, was to remain supreme. +The monopoly robber-barons of those other industries +were humbled and shattered by government trust-busting. + +Vail, the former Post Office official, was quite willing +to accommodate the US government; in fact he would +forge an active alliance with it. AT&T would become +almost a wing of the American government, almost +another Post Office--though not quite. AT&T would +willingly submit to federal regulation, but in return, +it would use the government's regulators as its own police, +who would keep out competitors and assure the Bell +system's profits and preeminence. + +This was the second birth--the political birth--of the +American telephone system. Vail's arrangement was to +persist, with vast success, for many decades, until 1982. +His system was an odd kind of American industrial socialism. +It was born at about the same time as Leninist Communism, +and it lasted almost as long--and, it must be admitted, +to considerably better effect. + +Vail's system worked. Except perhaps for aerospace, +there has been no technology more thoroughly dominated +by Americans than the telephone. The telephone was +seen from the beginning as a quintessentially American +technology. Bell's policy, and the policy of Theodore Vail, +was a profoundly democratic policy of UNIVERSAL ACCESS. +Vail's famous corporate slogan, "One Policy, One System, +Universal Service," was a political slogan, with a very +American ring to it. + +The American telephone was not to become the specialized tool +of government or business, but a general public utility. +At first, it was true, only the wealthy could afford +private telephones, and Bell's company pursued the +business markets primarily. The American phone system +was a capitalist effort, meant to make money; it was not a charity. +But from the first, almost all communities with telephone service +had public telephones. And many stores--especially drugstores-- +offered public use of their phones. You might not own a telephone-- +but you could always get into the system, if you really needed to. + +There was nothing inevitable about this decision to make telephones +"public" and "universal." Vail's system involved a profound act +of trust in the public. This decision was a political one, +informed by the basic values of the American republic. +The situation might have been very different; +and in other countries, under other systems, +it certainly was. + +Joseph Stalin, for instance, vetoed plans for a Soviet +phone system soon after the Bolshevik revolution. +Stalin was certain that publicly accessible telephones +would become instruments of anti-Soviet counterrevolution +and conspiracy. (He was probably right.) When telephones +did arrive in the Soviet Union, they would be instruments +of Party authority, and always heavily tapped. (Alexander +Solzhenitsyn's prison-camp novel The First Circle +describes efforts to develop a phone system more suited +to Stalinist purposes.) + +France, with its tradition of rational centralized government, +had fought bitterly even against the electric telegraph, +which seemed to the French entirely too anarchical and frivolous. +For decades, nineteenth-century France communicated via the +"visual telegraph," a nation-spanning, government-owned semaphore +system of huge stone towers that signalled from hilltops, +across vast distances, with big windmill-like arms. +In 1846, one Dr. Barbay, a semaphore enthusiast, +memorably uttered an early version of what might be called +"the security expert's argument" against the open media. + +"No, the electric telegraph is not a sound invention. +It will always be at the mercy of the slightest disruption, +wild youths, drunkards, bums, etc. . . . The electric telegraph +meets those destructive elements with only a few meters of wire +over which supervision is impossible. A single man could, +without being seen, cut the telegraph wires leading to Paris, +and in twenty-four hours cut in ten different places the wires +of the same line, without being arrested. The visual telegraph, +on the contrary, has its towers, its high walls, its gates +well-guarded from inside by strong armed men. Yes, I declare, +substitution of the electric telegraph for the visual one +is a dreadful measure, a truly idiotic act." + +Dr. Barbay and his high-security stone machines +were eventually unsuccessful, but his argument-- +that communication exists for the safety and convenience +of the state, and must be carefully protected from the wild +boys and the gutter rabble who might want to crash the +system--would be heard again and again. + +When the French telephone system finally did arrive, +its snarled inadequacy was to be notorious. Devotees +of the American Bell System often recommended a trip +to France, for skeptics. + +In Edwardian Britain, issues of class and privacy +were a ball-and-chain for telephonic progress. It was +considered outrageous that anyone--any wild fool off +the street--could simply barge bellowing into one's office +or home, preceded only by the ringing of a telephone bell. +In Britain, phones were tolerated for the use of business, +but private phones tended be stuffed away into closets, +smoking rooms, or servants' quarters. Telephone operators +were resented in Britain because they did not seem to +"know their place." And no one of breeding would print +a telephone number on a business card; this seemed a crass +attempt to make the acquaintance of strangers. + +But phone access in America was to become a popular right; +something like universal suffrage, only more so. +American women could not yet vote when the phone system +came through; yet from the beginning American women +doted on the telephone. This "feminization" of the +American telephone was often commented on by foreigners. +Phones in America were not censored or stiff or formalized; +they were social, private, intimate, and domestic. +In America, Mother's Day is by far the busiest day +of the year for the phone network. + +The early telephone companies, and especially AT&T, +were among the foremost employers of American women. +They employed the daughters of the American middle-class +in great armies: in 1891, eight thousand women; by 1946, +almost a quarter of a million. Women seemed to enjoy +telephone work; it was respectable, it was steady, +it paid fairly well as women's work went, and--not least-- +it seemed a genuine contribution to the social good +of the community. Women found Vail's ideal of public +service attractive. This was especially true in rural areas, +where women operators, running extensive rural party-lines, +enjoyed considerable social power. The operator knew everyone +on the party-line, and everyone knew her. + +Although Bell himself was an ardent suffragist, the +telephone company did not employ women for the sake of +advancing female liberation. AT&T did this for sound +commercial reasons. The first telephone operators of +the Bell system were not women, but teenage American boys. +They were telegraphic messenger boys (a group about to +be rendered technically obsolescent), who swept up +around the phone office, dunned customers for bills, +and made phone connections on the switchboard, +all on the cheap. + +Within the very first year of operation, 1878, +Bell's company learned a sharp lesson about combining +teenage boys and telephone switchboards. Putting +teenage boys in charge of the phone system brought swift +and consistent disaster. Bell's chief engineer described them +as "Wild Indians." The boys were openly rude to customers. +They talked back to subscribers, saucing off, +uttering facetious remarks, and generally giving lip. +The rascals took Saint Patrick's Day off without permission. +And worst of all they played clever tricks with +the switchboard plugs: disconnecting calls, crossing lines +so that customers found themselves talking to strangers, +and so forth. + +This combination of power, technical mastery, and effective +anonymity seemed to act like catnip on teenage boys. + +This wild-kid-on-the-wires phenomenon was not confined to +the USA; from the beginning, the same was true of the British +phone system. An early British commentator kindly remarked: +"No doubt boys in their teens found the work not a little irksome, +and it is also highly probable that under the early conditions +of employment the adventurous and inquisitive spirits of which +the average healthy boy of that age is possessed, were not always +conducive to the best attention being given to the wants +of the telephone subscribers." + +So the boys were flung off the system--or at least, +deprived of control of the switchboard. But the +"adventurous and inquisitive spirits" of the teenage boys +would be heard from in the world of telephony, again and again. + +The fourth stage in the technological life-cycle is death: +"the Dog," dead tech. The telephone has so far avoided this fate. +On the contrary, it is thriving, still spreading, still evolving, +and at increasing speed. + +The telephone has achieved a rare and exalted state for a +technological artifact: it has become a HOUSEHOLD OBJECT. +The telephone, like the clock, like pen and paper, +like kitchen utensils and running water, has become +a technology that is visible only by its absence. +The telephone is technologically transparent. +The global telephone system is the largest and most +complex machine in the world, yet it is easy to use. +More remarkable yet, the telephone is almost entirely +physically safe for the user. + +For the average citizen in the 1870s, the telephone +was weirder, more shocking, more "high-tech" and +harder to comprehend, than the most outrageous stunts +of advanced computing for us Americans in the 1990s. +In trying to understand what is happening to us today, +with our bulletin-board systems, direct overseas dialling, +fiber-optic transmissions, computer viruses, hacking stunts, +and a vivid tangle of new laws and new crimes, it is important +to realize that our society has been through a similar challenge before-- +and that, all in all, we did rather well by it. + +Bell's stage telephone seemed bizarre at first. But the +sensations of weirdness vanished quickly, once people began +to hear the familiar voices of relatives and friends, +in their own homes on their own telephones. The telephone +changed from a fearsome high-tech totem to an everyday pillar +of human community. + +This has also happened, and is still happening, +to computer networks. Computer networks such as +NSFnet, BITnet, USENET, JANET, are technically +advanced, intimidating, and much harder to use than +telephones. Even the popular, commercial computer +networks, such as GEnie, Prodigy, and CompuServe, +cause much head-scratching and have been described +as "user-hateful." Nevertheless they too are changing +from fancy high-tech items into everyday sources +of human community. + +The words "community" and "communication" have +the same root. Wherever you put a communications +network, you put a community as well. And whenever +you TAKE AWAY that network--confiscate it, outlaw it, +crash it, raise its price beyond affordability-- +then you hurt that community. + +Communities will fight to defend themselves. People will fight harder +and more bitterly to defend their communities, than they will fight +to defend their own individual selves. And this is very true +of the "electronic community" that arose around computer networks +in the 1980s--or rather, the VARIOUS electronic communities, +in telephony, law enforcement, computing, and the digital +underground that, by the year 1990, were raiding, rallying, +arresting, suing, jailing, fining and issuing angry manifestos. + +None of the events of 1990 were entirely new. +Nothing happened in 1990 that did not have some kind +of earlier and more understandable precedent. What gave +the Hacker Crackdown its new sense of gravity and +importance was the feeling--the COMMUNITY feeling-- +that the political stakes had been raised; that trouble +in cyberspace was no longer mere mischief or inconclusive +skirmishing, but a genuine fight over genuine issues, +a fight for community survival and the shape of the future. + +These electronic communities, having flourished throughout +the 1980s, were becoming aware of themselves, and increasingly, +becoming aware of other, rival communities. Worries were +sprouting up right and left, with complaints, rumors, +uneasy speculations. But it would take a catalyst, a shock, +to make the new world evident. Like Bell's great publicity break, +the Tarriffville Rail Disaster of January 1878, +it would take a cause celebre. + +That cause was the AT&T Crash of January 15, 1990. +After the Crash, the wounded and anxious telephone +community would come out fighting hard. + +# + +The community of telephone technicians, engineers, operators +and researchers is the oldest community in cyberspace. +These are the veterans, the most developed group, +the richest, the most respectable, in most ways the most powerful. +Whole generations have come and gone since Alexander Graham Bell's day, +but the community he founded survives; people work for the phone system +today whose great-grandparents worked for the phone system. +Its specialty magazines, such as Telephony, AT&T Technical Journal, +Telephone Engineer and Management, are decades old; +they make computer publications like Macworld and PC Week +look like amateur johnny-come-latelies. + +And the phone companies take no back seat in high-technology, either. +Other companies' industrial researchers may have won new markets; +but the researchers of Bell Labs have won SEVEN NOBEL PRIZES. +One potent device that Bell Labs originated, the transistor, +has created entire GROUPS of industries. Bell Labs are +world-famous for generating "a patent a day," and have even +made vital discoveries in astronomy, physics and cosmology. + +Throughout its seventy-year history, "Ma Bell" was not so much +a company as a way of life. Until the cataclysmic divestiture +of the 1980s, Ma Bell was perhaps the ultimate maternalist mega-employer. +The AT&T corporate image was the "gentle giant," "the voice with a smile," +a vaguely socialist-realist world of cleanshaven linemen in shiny helmets +and blandly pretty phone-girls in headsets and nylons. Bell System +employees were famous as rock-ribbed Kiwanis and Rotary members, +Little-League enthusiasts, school-board people. + +During the long heyday of Ma Bell, the Bell employee corps +were nurtured top-to-bottom on a corporate ethos of public service. +There was good money in Bell, but Bell was not ABOUT money; +Bell used public relations, but never mere marketeering. +People went into the Bell System for a good life, +and they had a good life. But it was not mere money +that led Bell people out in the midst of storms and earthquakes +to fight with toppled phone-poles, to wade in flooded manholes, +to pull the red-eyed graveyard-shift over collapsing switching-systems. +The Bell ethic was the electrical equivalent of the postman's: +neither rain, nor snow, nor gloom of night would stop these couriers. + +It is easy to be cynical about this, as it is easy to be +cynical about any political or social system; but cynicism +does not change the fact that thousands of people took +these ideals very seriously. And some still do. + +The Bell ethos was about public service; and that was +gratifying; but it was also about private POWER, and that +was gratifying too. As a corporation, Bell was very special. +Bell was privileged. Bell had snuggled up close to the state. +In fact, Bell was as close to government as you could get in +America and still make a whole lot of legitimate money. + +But unlike other companies, Bell was above and beyond +the vulgar commercial fray. Through its regional operating companies, +Bell was omnipresent, local, and intimate, all over America; +but the central ivory towers at its corporate heart were the +tallest and the ivoriest around. + +There were other phone companies in America, to be sure; +the so-called independents. Rural cooperatives, mostly; +small fry, mostly tolerated, sometimes warred upon. +For many decades, "independent" American phone companies +lived in fear and loathing of the official Bell monopoly +(or the "Bell Octopus," as Ma Bell's nineteenth-century +enemies described her in many angry newspaper manifestos). +Some few of these independent entrepreneurs, while legally +in the wrong, fought so bitterly against the Octopus +that their illegal phone networks were cast into the street +by Bell agents and publicly burned. + +The pure technical sweetness of the Bell System gave its operators, +inventors and engineers a deeply satisfying sense of power and mastery. +They had devoted their lives to improving this vast nation-spanning machine; +over years, whole human lives, they had watched it improve and grow. +It was like a great technological temple. They were an elite, +and they knew it--even if others did not; in fact, they felt +even more powerful BECAUSE others did not understand. + +The deep attraction of this sensation of elite technical power +should never be underestimated. "Technical power" is not for everybody; +for many people it simply has no charm at all. But for some people, +it becomes the core of their lives. For a few, it is overwhelming, +obsessive; it becomes something close to an addiction. People--especially +clever teenage boys whose lives are otherwise mostly powerless and put-upon +--love this sensation of secret power, and are willing to do all sorts +of amazing things to achieve it. The technical POWER of electronics +has motivated many strange acts detailed in this book, which would +otherwise be inexplicable. + +So Bell had power beyond mere capitalism. The Bell service ethos worked, +and was often propagandized, in a rather saccharine fashion. Over the decades, +people slowly grew tired of this. And then, openly impatient with it. +By the early 1980s, Ma Bell was to find herself with scarcely a real friend +in the world. Vail's industrial socialism had become hopelessly +out-of-fashion politically. Bell would be punished for that. +And that punishment would fall harshly upon the people of the +telephone community. + +# + +In 1983, Ma Bell was dismantled by federal court action. +The pieces of Bell are now separate corporate entities. +The core of the company became AT&T Communications, +and also AT&T Industries (formerly Western Electric, +Bell's manufacturing arm). AT&T Bell Labs became Bell +Communications Research, Bellcore. Then there are the +Regional Bell Operating Companies, or RBOCs, pronounced "arbocks." + +Bell was a titan and even these regional chunks are gigantic enterprises: +Fortune 50 companies with plenty of wealth and power behind them. +But the clean lines of "One Policy, One System, Universal Service" +have been shattered, apparently forever. + +The "One Policy" of the early Reagan Administration was to +shatter a system that smacked of noncompetitive socialism. +Since that time, there has been no real telephone "policy" +on the federal level. Despite the breakup, the remnants +of Bell have never been set free to compete in the open marketplace. + +The RBOCs are still very heavily regulated, but not from the top. +Instead, they struggle politically, economically and legally, +in what seems an endless turmoil, in a patchwork of overlapping federal +and state jurisdictions. Increasingly, like other major American corporations, +the RBOCs are becoming multinational, acquiring important commercial interests +in Europe, Latin America, and the Pacific Rim. But this, too, adds to their +legal and political predicament. + +The people of what used to be Ma Bell are not happy about their fate. +They feel ill-used. They might have been grudgingly willing to make +a full transition to the free market; to become just companies amid +other companies. But this never happened. Instead, AT&T and the RBOCS +("the Baby Bells") feel themselves wrenched from side to side by state +regulators, by Congress, by the FCC, and especially by the federal court +of Judge Harold Greene, the magistrate who ordered the Bell breakup +and who has been the de facto czar of American telecommunications +ever since 1983. + +Bell people feel that they exist in a kind of paralegal limbo today. +They don't understand what's demanded of them. If it's "service," +why aren't they treated like a public service? And if it's money, +then why aren't they free to compete for it? No one seems to know, +really. Those who claim to know keep changing their minds. +Nobody in authority seems willing to grasp the nettle for once and all. + +Telephone people from other countries are amazed by the +American telephone system today. Not that it works so well; +for nowadays even the French telephone system works, more or less. +They are amazed that the American telephone system STILL works +AT ALL, under these strange conditions. + +Bell's "One System" of long-distance service is now only about +eighty percent of a system, with the remainder held by Sprint, MCI, +and the midget long-distance companies. Ugly wars over dubious +corporate practices such as "slamming" (an underhanded method +of snitching clients from rivals) break out with some regularity +in the realm of long-distance service. The battle to break Bell's +long-distance monopoly was long and ugly, and since the breakup +the battlefield has not become much prettier. AT&T's famous +shame-and-blame advertisements, which emphasized the shoddy work +and purported ethical shadiness of their competitors, were much +remarked on for their studied psychological cruelty. + +There is much bad blood in this industry, and much +long-treasured resentment. AT&T's post-breakup +corporate logo, a striped sphere, is known in the +industry as the "Death Star" (a reference from the movie +Star Wars, in which the "Death Star" was the spherical +high- tech fortress of the harsh-breathing imperial ultra-baddie, +Darth Vader.) Even AT&T employees are less than thrilled +by the Death Star. A popular (though banned) T-shirt among +AT&T employees bears the old-fashioned Bell logo of the Bell System, +plus the newfangled striped sphere, with the before-and-after comments: +"This is your brain--This is your brain on drugs!" AT&T made a very +well-financed and determined effort to break into the personal +computer market; it was disastrous, and telco computer experts +are derisively known by their competitors as "the pole-climbers." +AT&T and the Baby Bell arbocks still seem to have few friends. + +Under conditions of sharp commercial competition, a crash like +that of January 15, 1990 was a major embarrassment to AT&T. +It was a direct blow against their much-treasured reputation +for reliability. Within days of the crash AT&T's +Chief Executive Officer, Bob Allen, officially apologized, +in terms of deeply pained humility: + +"AT&T had a major service disruption last Monday. +We didn't live up to our own standards of quality, +and we didn't live up to yours. It's as simple as that. +And that's not acceptable to us. Or to you. . . . +We understand how much people have come to depend +upon AT&T service, so our AT&T Bell Laboratories scientists +and our network engineers are doing everything possible +to guard against a recurrence. . . . We know there's no way +to make up for the inconvenience this problem may have caused you." + +Mr Allen's "open letter to customers" was printed in lavish ads +all over the country: in the Wall Street Journal, USA Today, +New York Times, Los Angeles Times, Chicago Tribune, +Philadelphia Inquirer, San Francisco Chronicle Examiner, +Boston Globe, Dallas Morning News, Detroit Free Press, +Washington Post, Houston Chronicle, Cleveland Plain Dealer, +Atlanta Journal Constitution, Minneapolis Star Tribune, +St. Paul Pioneer Press Dispatch, Seattle Times/Post Intelligencer, +Tacoma News Tribune, Miami Herald, Pittsburgh Press, +St. Louis Post Dispatch, Denver Post, Phoenix Republic Gazette +and Tampa Tribune. + +In another press release, AT&T went to some pains to suggest +that this "software glitch" might have happened just as easily to MCI, +although, in fact, it hadn't. (MCI's switching software was quite different +from AT&T's--though not necessarily any safer.) AT&T also announced +their plans to offer a rebate of service on Valentine's Day to make up +for the loss during the Crash. + +"Every technical resource available, including Bell Labs +scientists and engineers, has been devoted to assuring +it will not occur again," the public was told. They were +further assured that "The chances of a recurrence are small-- +a problem of this magnitude never occurred before." + +In the meantime, however, police and corporate +security maintained their own suspicions about +"the chances of recurrence" and the real reason why +a "problem of this magnitude" had appeared, seemingly +out of nowhere. Police and security knew for a fact +that hackers of unprecedented sophistication were illegally +entering, and reprogramming, certain digital switching stations. +Rumors of hidden "viruses" and secret "logic bombs" +in the switches ran rampant in the underground, +with much chortling over AT&T's predicament, +and idle speculation over what unsung hacker genius +was responsible for it. Some hackers, including police +informants, were trying hard to finger one another +as the true culprits of the Crash. + +Telco people found little comfort in objectivity when +they contemplated these possibilities. It was just too close +to the bone for them; it was embarrassing; it hurt so much, +it was hard even to talk about. + +There has always been thieving and misbehavior in the phone system. +There has always been trouble with the rival independents, +and in the local loops. But to have such trouble in the core +of the system, the long-distance switching stations, +is a horrifying affair. To telco people, this is +all the difference between finding roaches in your kitchen +and big horrid sewer-rats in your bedroom. + +From the outside, to the average citizen, the telcos +still seem gigantic and impersonal. The American public +seems to regard them as something akin to Soviet apparats. +Even when the telcos do their best corporate-citizen routine, +subsidizing magnet high-schools and sponsoring news-shows +on public television, they seem to win little except public suspicion. + +But from the inside, all this looks very different. +There's harsh competition. A legal and political system +that seems baffled and bored, when not actively hostile +to telco interests. There's a loss of morale, a deep sensation +of having somehow lost the upper hand. Technological change +has caused a loss of data and revenue to other, newer forms +of transmission. There's theft, and new forms of theft, +of growing scale and boldness and sophistication. +With all these factors, it was no surprise to see the telcos, +large and small, break out in a litany of bitter complaint. + +In late '88 and throughout 1989, telco representatives +grew shrill in their complaints to those few American law +enforcement officials who make it their business to try to +understand what telephone people are talking about. +Telco security officials had discovered the computer- +hacker underground, infiltrated it thoroughly, +and become deeply alarmed at its growing expertise. +Here they had found a target that was not only loathsome +on its face, but clearly ripe for counterattack. + +Those bitter rivals: AT&T, MCI and Sprint--and a crowd +of Baby Bells: PacBell, Bell South, Southwestern Bell, +NYNEX, USWest, as well as the Bell research consortium Bellcore, +and the independent long-distance carrier Mid-American-- +all were to have their role in the great hacker dragnet of 1990. +After years of being battered and pushed around, the telcos had, +at least in a small way, seized the initiative again. +After years of turmoil, telcos and government officials were +once again to work smoothly in concert in defense of the System. +Optimism blossomed; enthusiasm grew on all sides; +the prospective taste of vengeance was sweet. + +# + +From the beginning--even before the crackdown had a name-- +secrecy was a big problem. There were many good reasons +for secrecy in the hacker crackdown. Hackers and code-thieves +were wily prey, slinking back to their bedrooms and basements +and destroying vital incriminating evidence at the first hint of trouble. +Furthermore, the crimes themselves were heavily technical and difficult +to describe, even to police--much less to the general public. + +When such crimes HAD been described intelligibly to the public, +in the past, that very publicity had tended to INCREASE the crimes +enormously. Telco officials, while painfully aware of the vulnerabilities +of their systems, were anxious not to publicize those weaknesses. +Experience showed them that those weaknesses, once discovered, +would be pitilessly exploited by tens of thousands of people--not only +by professional grifters and by underground hackers and phone phreaks, +but by many otherwise more-or-less honest everyday folks, who regarded +stealing service from the faceless, soulless "Phone Company" as a kind of +harmless indoor sport. When it came to protecting their interests, +telcos had long since given up on general public sympathy for +"the Voice with a Smile." Nowadays the telco's "Voice" was +very likely to be a computer's; and the American public +showed much less of the proper respect and gratitude due +the fine public service bequeathed them by Dr. Bell and Mr. Vail. +The more efficient, high-tech, computerized, and impersonal +the telcos became, it seemed, the more they were met by +sullen public resentment and amoral greed. + +Telco officials wanted to punish the phone-phreak underground, in as +public and exemplary a manner as possible. They wanted to make dire +examples of the worst offenders, to seize the ringleaders and intimidate +the small fry, to discourage and frighten the wacky hobbyists, and send +the professional grifters to jail. To do all this, publicity was vital. + +Yet operational secrecy was even more so. If word got out that +a nationwide crackdown was coming, the hackers might simply vanish; +destroy the evidence, hide their computers, go to earth, +and wait for the campaign to blow over. Even the young +hackers were crafty and suspicious, and as for the professional grifters, +they tended to split for the nearest state-line at the first sign of trouble. +For the crackdown to work well, they would all have to be caught red-handed, +swept upon suddenly, out of the blue, from every corner of the compass. + +And there was another strong motive for secrecy. In the worst-case scenario, +a blown campaign might leave the telcos open to a devastating hacker +counter-attack. If there were indeed hackers loose in America who +had caused the January 15 Crash--if there were truly gifted hackers, +loose in the nation's long-distance switching systems, and enraged +or frightened by the crackdown--then they might react unpredictably +to an attempt to collar them. Even if caught, they might have talented +and vengeful friends still running around loose. Conceivably, +it could turn ugly. Very ugly. In fact, it was hard to imagine +just how ugly things might turn, given that possibility. + +Counter-attack from hackers was a genuine concern for the telcos. +In point of fact, they would never suffer any such counter-attack. +But in months to come, they would be at some pains to publicize +this notion and to utter grim warnings about it. + +Still, that risk seemed well worth running. Better to run the risk +of vengeful attacks, than to live at the mercy of potential crashers. +Any cop would tell you that a protection racket had no real future. + +And publicity was such a useful thing. Corporate security officers, +including telco security, generally work under conditions of great discretion. +And corporate security officials do not make money for their companies. +Their job is to PREVENT THE LOSS of money, which is much less glamorous +than actually winning profits. + +If you are a corporate security official, and you do your job brilliantly, +then nothing bad happens to your company at all. Because of this, you appear +completely superfluous. This is one of the many unattractive aspects +of security work. It's rare that these folks have the chance to draw +some healthy attention to their own efforts. + +Publicity also served the interest of their friends in law enforcement. +Public officials, including law enforcement officials, thrive by attracting +favorable public interest. A brilliant prosecution in a matter of vital +public interest can make the career of a prosecuting attorney. +And for a police officer, good publicity opens the purses of the legislature; +it may bring a citation, or a promotion, or at least a rise in status +and the respect of one's peers. + +But to have both publicity and secrecy is to have one's cake and eat it too. +In months to come, as we will show, this impossible act was to cause great +pain to the agents of the crackdown. But early on, it seemed possible +--maybe even likely--that the crackdown could successfully combine +the best of both worlds. The ARREST of hackers would be heavily publicized. +The actual DEEDS of the hackers, which were technically hard to explain +and also a security risk, would be left decently obscured. The THREAT +hackers posed would be heavily trumpeted; the likelihood of their actually +committing such fearsome crimes would be left to the public's imagination. +The spread of the computer underground, and its growing technical +sophistication, would be heavily promoted; the actual hackers themselves, +mostly bespectacled middle-class white suburban teenagers, +would be denied any personal publicity. + +It does not seem to have occurred to any telco official +that the hackers accused would demand a day in court; +that journalists would smile upon the hackers as +"good copy;" that wealthy high-tech entrepreneurs would +offer moral and financial support to crackdown victims; +that constitutional lawyers would show up with briefcases, +frowning mightily. This possibility does not seem to have +ever entered the game-plan. + +And even if it had, it probably would not have slowed +the ferocious pursuit of a stolen phone-company document, +mellifluously known as "Control Office Administration of +Enhanced 911 Services for Special Services and Major Account Centers." + +In the chapters to follow, we will explore the worlds +of police and the computer underground, and the large +shadowy area where they overlap. But first, we must +explore the battleground. Before we leave the world +of the telcos, we must understand what a switching system +actually is and how your telephone actually works. + +# + +To the average citizen, the idea of the telephone is represented by, +well, a TELEPHONE: a device that you talk into. To a telco +professional, however, the telephone itself is known, in lordly +fashion, as a "subset." The "subset" in your house is a mere adjunct, +a distant nerve ending, of the central switching stations, +which are ranked in levels of heirarchy, up to the long-distance electronic +switching stations, which are some of the largest computers on earth. + +Let us imagine that it is, say, 1925, before the +introduction of computers, when the phone system was +simpler and somewhat easier to grasp. Let's further +imagine that you are Miss Leticia Luthor, a fictional +operator for Ma Bell in New York City of the 20s. + +Basically, you, Miss Luthor, ARE the "switching system." +You are sitting in front of a large vertical switchboard, +known as a "cordboard," made of shiny wooden panels, +with ten thousand metal-rimmed holes punched in them, +known as jacks. The engineers would have put more +holes into your switchboard, but ten thousand is +as many as you can reach without actually having +to get up out of your chair. + +Each of these ten thousand holes has its own little electric lightbulb, +known as a "lamp," and its own neatly printed number code. + +With the ease of long habit, you are scanning your board for lit-up bulbs. +This is what you do most of the time, so you are used to it. + +A lamp lights up. This means that the phone +at the end of that line has been taken off the hook. +Whenever a handset is taken off the hook, that closes a circuit +inside the phone which then signals the local office, i.e. you, +automatically. There might be somebody calling, or then +again the phone might be simply off the hook, but this +does not matter to you yet. The first thing you do, +is record that number in your logbook, in your fine American +public-school handwriting. This comes first, naturally, +since it is done for billing purposes. + +You now take the plug of your answering cord, which goes +directly to your headset, and plug it into the lit-up hole. +"Operator," you announce. + +In operator's classes, before taking this job, you have +been issued a large pamphlet full of canned operator's +responses for all kinds of contingencies, which you had +to memorize. You have also been trained in a proper +non-regional, non-ethnic pronunciation and tone of voice. +You rarely have the occasion to make any spontaneous +remark to a customer, and in fact this is frowned upon +(except out on the rural lines where people have time +on their hands and get up to all kinds of mischief). + +A tough-sounding user's voice at the end of the line +gives you a number. Immediately, you write that number +down in your logbook, next to the caller's number, +which you just wrote earlier. You then look and see if +the number this guy wants is in fact on your switchboard, +which it generally is, since it's generally a local call. +Long distance costs so much that people use it sparingly. + +Only then do you pick up a calling-cord from a shelf +at the base of the switchboard. This is a long elastic cord +mounted on a kind of reel so that it will zip back in when +you unplug it. There are a lot of cords down there, +and when a bunch of them are out at once they look like +a nest of snakes. Some of the girls think there are bugs +living in those cable-holes. They're called "cable mites" +and are supposed to bite your hands and give you rashes. +You don't believe this, yourself. + +Gripping the head of your calling-cord, you slip the tip +of it deftly into the sleeve of the jack for the called person. +Not all the way in, though. You just touch it. If you hear +a clicking sound, that means the line is busy and you can't +put the call through. If the line is busy, you have to stick +the calling-cord into a "busy-tone jack," which will give +the guy a busy-tone. This way you don't have to talk to him +yourself and absorb his natural human frustration. + +But the line isn't busy. So you pop the cord all the way in. +Relay circuits in your board make the distant phone ring, +and if somebody picks it up off the hook, then a phone +conversation starts. You can hear this conversation +on your answering cord, until you unplug it. In fact +you could listen to the whole conversation if you wanted, +but this is sternly frowned upon by management, and frankly, +when you've overheard one, you've pretty much heard 'em all. + +You can tell how long the conversation lasts by the glow +of the calling-cord's lamp, down on the calling-cord's shelf. +When it's over, you unplug and the calling-cord zips back into place. + +Having done this stuff a few hundred thousand times, +you become quite good at it. In fact you're plugging, +and connecting, and disconnecting, ten, twenty, forty cords +at a time. It's a manual handicraft, really, quite satisfying +in a way, rather like weaving on an upright loom. + +Should a long-distance call come up, it would be different, +but not all that different. Instead of connecting the call +through your own local switchboard, you have to go up the hierarchy, +onto the long-distance lines, known as "trunklines." +Depending on how far the call goes, it may have to work +its way through a whole series of operators, which can +take quite a while. The caller doesn't wait on the line +while this complex process is negotiated across the country +by the gaggle of operators. Instead, the caller hangs up, +and you call him back yourself when the call has finally +worked its way through. + +After four or five years of this work, you get married, +and you have to quit your job, this being the natural order +of womanhood in the American 1920s. The phone company +has to train somebody else--maybe two people, since +the phone system has grown somewhat in the meantime. +And this costs money. + +In fact, to use any kind of human being as a switching +system is a very expensive proposition. Eight thousand +Leticia Luthors would be bad enough, but a quarter of a +million of them is a military-scale proposition and makes +drastic measures in automation financially worthwhile. + +Although the phone system continues to grow today, +the number of human beings employed by telcos has +been dropping steadily for years. Phone "operators" +now deal with nothing but unusual contingencies, +all routine operations having been shrugged off onto machines. +Consequently, telephone operators are considerably less +machine-like nowadays, and have been known to have accents +and actual character in their voices. When you reach +a human operator today, the operators are rather more +"human" than they were in Leticia's day--but on the other hand, +human beings in the phone system are much harder to reach +in the first place. + +Over the first half of the twentieth century, +"electromechanical" switching systems of growing +complexity were cautiously introduced into the phone system. +In certain backwaters, some of these hybrid systems are still +in use. But after 1965, the phone system began to go completely +electronic, and this is by far the dominant mode today. +Electromechanical systems have "crossbars," and "brushes," +and other large moving mechanical parts, which, while faster +and cheaper than Leticia, are still slow, and tend to wear out +fairly quickly. + +But fully electronic systems are inscribed on silicon chips, +and are lightning-fast, very cheap, and quite durable. +They are much cheaper to maintain than even the best +electromechanical systems, and they fit into half the space. +And with every year, the silicon chip grows smaller, faster, +and cheaper yet. Best of all, automated electronics work +around the clock and don't have salaries or health insurance. + +There are, however, quite serious drawbacks to the +use of computer-chips. When they do break down, it is +a daunting challenge to figure out what the heck has gone +wrong with them. A broken cordboard generally had +a problem in it big enough to see. A broken chip has +invisible, microscopic faults. And the faults in bad +software can be so subtle as to be practically theological. + +If you want a mechanical system to do something new, +then you must travel to where it is, and pull pieces out of it, +and wire in new pieces. This costs money. However, if you want +a chip to do something new, all you have to do is change its software, +which is easy, fast and dirt-cheap. You don't even have to see the chip +to change its program. Even if you did see the chip, it wouldn't look +like much. A chip with program X doesn't look one whit different from +a chip with program Y. + +With the proper codes and sequences, and access to specialized phone-lines, +you can change electronic switching systems all over America from anywhere +you please. + +And so can other people. If they know how, and if they want to, +they can sneak into a microchip via the special phonelines and diddle with it, +leaving no physical trace at all. If they broke into the operator's station +and held Leticia at gunpoint, that would be very obvious. If they broke into +a telco building and went after an electromechanical switch with a toolbelt, +that would at least leave many traces. But people can do all manner of amazing +things to computer switches just by typing on a keyboard, and keyboards are +everywhere today. The extent of this vulnerability is deep, dark, broad, +almost mind-boggling, and yet this is a basic, primal fact of life about +any computer on a network. + +Security experts over the past twenty years have insisted, +with growing urgency, that this basic vulnerability of computers +represents an entirely new level of risk, of unknown but obviously +dire potential to society. And they are right. + +An electronic switching station does pretty much +everything Letitia did, except in nanoseconds and +on a much larger scale. Compared to Miss Luthor's +ten thousand jacks, even a primitive 1ESS switching computer, +60s vintage, has a 128,000 lines. And the current AT&T +system of choice is the monstrous fifth-generation 5ESS. + +An Electronic Switching Station can scan every line on its "board" +in a tenth of a second, and it does this over and over, tirelessly, +around the clock. Instead of eyes, it uses "ferrod scanners" +to check the condition of local lines and trunks. Instead of hands, +it has "signal distributors," "central pulse distributors," +"magnetic latching relays," and "reed switches," which complete +and break the calls. Instead of a brain, it has a "central processor." +Instead of an instruction manual, it has a program. Instead of +a handwritten logbook for recording and billing calls, +it has magnetic tapes. And it never has to talk to anybody. +Everything a customer might say to it is done by punching +the direct-dial tone buttons on your subset. + +Although an Electronic Switching Station can't talk, +it does need an interface, some way to relate to its, er, +employers. This interface is known as the "master control +center." (This interface might be better known simply as +"the interface," since it doesn't actually "control" phone +calls directly. However, a term like "Master Control +Center" is just the kind of rhetoric that telco maintenance +engineers--and hackers--find particularly satisfying.) + +Using the master control center, a phone engineer can test +local and trunk lines for malfunctions. He (rarely she) +can check various alarm displays, measure traffic on the lines, +examine the records of telephone usage and the charges for those calls, +and change the programming. + +And, of course, anybody else who gets into the master control center +by remote control can also do these things, if he (rarely she) +has managed to figure them out, or, more likely, has somehow swiped +the knowledge from people who already know. + +In 1989 and 1990, one particular RBOC, BellSouth, +which felt particularly troubled, spent a purported $1.2 +million on computer security. Some think it spent as +much as two million, if you count all the associated costs. +Two million dollars is still very little compared to the +great cost-saving utility of telephonic computer systems. + +Unfortunately, computers are also stupid. +Unlike human beings, computers possess the truly +profound stupidity of the inanimate. + +In the 1960s, in the first shocks of spreading computerization, +there was much easy talk about the stupidity of computers-- +how they could "only follow the program" and were rigidly required +to do "only what they were told." There has been rather less talk +about the stupidity of computers since they began to achieve +grandmaster status in chess tournaments, and to manifest +many other impressive forms of apparent cleverness. + +Nevertheless, computers STILL are profoundly brittle and stupid; +they are simply vastly more subtle in their stupidity and brittleness. +The computers of the 1990s are much more reliable in their components +than earlier computer systems, but they are also called upon to do +far more complex things, under far more challenging conditions. + +On a basic mathematical level, every single line of +a software program offers a chance for some possible screwup. +Software does not sit still when it works; it "runs," +it interacts with itself and with its own inputs and outputs. +By analogy, it stretches like putty into millions of possible +shapes and conditions, so many shapes that they can never +all be successfully tested, not even in the lifespan of the universe. +Sometimes the putty snaps. + +The stuff we call "software" is not like anything that human society +is used to thinking about. Software is something like a machine, +and something like mathematics, and something like language, and +something like thought, and art, and information. . . . But software +is not in fact any of those other things. The protean quality +of software is one of the great sources of its fascination. +It also makes software very powerful, very subtle, +very unpredictable, and very risky. + +Some software is bad and buggy. Some is "robust," +even "bulletproof." The best software is that which has +been tested by thousands of users under thousands of +different conditions, over years. It is then known as +"stable." This does NOT mean that the software is +now flawless, free of bugs. It generally means that there +are plenty of bugs in it, but the bugs are well-identified +and fairly well understood. + +There is simply no way to assure that software is free +of flaws. Though software is mathematical in nature, +it cannot by "proven" like a mathematical theorem; +software is more like language, with inherent ambiguities, +with different definitions, different assumptions, +different levels of meaning that can conflict. + +Human beings can manage, more or less, with +human language because we can catch the gist of it. + +Computers, despite years of effort in "artificial intelligence," +have proven spectacularly bad in "catching the gist" of anything at all. +The tiniest bit of semantic grit may still bring the mightiest computer +tumbling down. One of the most hazardous things you can do to a +computer program is try to improve it--to try to make it safer. +Software "patches" represent new, untried un-"stable" software, +which is by definition riskier. + +The modern telephone system has come to depend, +utterly and irretrievably, upon software. And the +System Crash of January 15, 1990, was caused by an +IMPROVEMENT in software. Or rather, an ATTEMPTED +improvement. + +As it happened, the problem itself--the problem per se--took this form. +A piece of telco software had been written in C language, a standard +language of the telco field. Within the C software was a +long "do. . .while" construct. The "do. . .while" construct +contained a "switch" statement. The "switch" statement contained +an "if" clause. The "if" clause contained a "break." The "break" +was SUPPOSED to "break" the "if clause." Instead, the "break" +broke the "switch" statement. + +That was the problem, the actual reason why people picking up phones +on January 15, 1990, could not talk to one another. + +Or at least, that was the subtle, abstract, cyberspatial +seed of the problem. This is how the problem manifested itself +from the realm of programming into the realm of real life. + +The System 7 software for AT&T's 4ESS switching station, +the "Generic 44E14 Central Office Switch Software," +had been extensively tested, and was considered very stable. +By the end of 1989, eighty of AT&T's switching systems +nationwide had been programmed with the new software. Cautiously, +thirty-four stations were left to run the slower, less-capable +System 6, because AT&T suspected there might be shakedown problems +with the new and unprecedently sophisticated System 7 network. + +The stations with System 7 were programmed to switch over to a backup net +in case of any problems. In mid-December 1989, however, a new high-velocity, +high-security software patch was distributed to each of the 4ESS switches +that would enable them to switch over even more quickly, making the System 7 +network that much more secure. + +Unfortunately, every one of these 4ESS switches was now in possession +of a small but deadly flaw. + +In order to maintain the network, switches must monitor +the condition of other switches--whether they are up and running, +whether they have temporarily shut down, whether they are overloaded +and in need of assistance, and so forth. The new software helped +control this bookkeeping function by monitoring the status calls +from other switches. + +It only takes four to six seconds for a troubled 4ESS switch +to rid itself of all its calls, drop everything temporarily, +and re-boot its software from scratch. Starting over from scratch +will generally rid the switch of any software problems that may have +developed in the course of running the system. Bugs that arise will +be simply wiped out by this process. It is a clever idea. This process +of automatically re-booting from scratch is known as the "normal fault +recovery routine." Since AT&T's software is in fact exceptionally stable, +systems rarely have to go into "fault recovery" in the first place; +but AT&T has always boasted of its "real world" reliability, and this +tactic is a belt-and-suspenders routine. + +The 4ESS switch used its new software to monitor its fellow switches +as they recovered from faults. As other switches came back on line +after recovery, they would send their "OK" signals to the switch. +The switch would make a little note to that effect in its "status map," +recognizing that the fellow switch was back and ready to go, +and should be sent some calls and put back to regular work. + +Unfortunately, while it was busy bookkeeping with the status map, +the tiny flaw in the brand-new software came into play. +The flaw caused the 4ESS switch to interact, subtly but drastically, +with incoming telephone calls from human users. If--and only if-- +two incoming phone-calls happened to hit the switch within a hundredth +of a second, then a small patch of data would be garbled by the flaw. + +But the switch had been programmed to monitor itself +constantly for any possible damage to its data. +When the switch perceived that its data had been somehow garbled, +then it too would go down, for swift repairs to its software. +It would signal its fellow switches not to send any more work. +It would go into the fault-recovery mode for four to six seconds. +And then the switch would be fine again, and would send out its "OK, +ready for work" signal. + +However, the "OK, ready for work" signal was the VERY THING THAT +HAD CAUSED THE SWITCH TO GO DOWN IN THE FIRST PLACE. And ALL the +System 7 switches had the same flaw in their status-map software. +As soon as they stopped to make the bookkeeping note that their fellow +switch was "OK," then they too would become vulnerable to the slight +chance that two phone-calls would hit them within a hundredth of a second. + +At approximately 2:25 P.M. EST on Monday, January 15, +one of AT&T's 4ESS toll switching systems in New York City +had an actual, legitimate, minor problem. It went into fault +recovery routines, announced "I'm going down," then announced, +"I'm back, I'm OK." And this cheery message then blasted +throughout the network to many of its fellow 4ESS switches. + +Many of the switches, at first, completely escaped trouble. +These lucky switches were not hit by the coincidence of +two phone calls within a hundredth of a second. +Their software did not fail--at first. But three switches-- +in Atlanta, St. Louis, and Detroit--were unlucky, +and were caught with their hands full. And they went down. +And they came back up, almost immediately. And they too began +to broadcast the lethal message that they, too, were "OK" again, +activating the lurking software bug in yet other switches. + +As more and more switches did have that bit of bad luck +and collapsed, the call-traffic became more and more densely +packed in the remaining switches, which were groaning +to keep up with the load. And of course, as the calls +became more densely packed, the switches were MUCH MORE LIKELY +to be hit twice within a hundredth of a second. + +It only took four seconds for a switch to get well. +There was no PHYSICAL damage of any kind to the switches, +after all. Physically, they were working perfectly. +This situation was "only" a software problem. + +But the 4ESS switches were leaping up and down every +four to six seconds, in a virulent spreading wave all over America, +in utter, manic, mechanical stupidity. They kept KNOCKING +one another down with their contagious "OK" messages. + +It took about ten minutes for the chain reaction to cripple the network. +Even then, switches would periodically luck-out and manage to resume +their normal work. Many calls--millions of them--were managing +to get through. But millions weren't. + +The switching stations that used System 6 were not directly affected. +Thanks to these old-fashioned switches, AT&T's national system avoided +complete collapse. This fact also made it clear to engineers that +System 7 was at fault. + +Bell Labs engineers, working feverishly in New Jersey, Illinois, +and Ohio, first tried their entire repertoire of standard network +remedies on the malfunctioning System 7. None of the remedies worked, +of course, because nothing like this had ever happened to any +phone system before. + +By cutting out the backup safety network entirely, +they were able to reduce the frenzy of "OK" messages +by about half. The system then began to recover, as the +chain reaction slowed. By 11:30 P.M. on Monday January +15, sweating engineers on the midnight shift breathed a +sigh of relief as the last switch cleared-up. + +By Tuesday they were pulling all the brand-new 4ESS software +and replacing it with an earlier version of System 7. + +If these had been human operators, rather than +computers at work, someone would simply have +eventually stopped screaming. It would have been +OBVIOUS that the situation was not "OK," and common +sense would have kicked in. Humans possess common sense-- +at least to some extent. Computers simply don't. + +On the other hand, computers can handle hundreds +of calls per second. Humans simply can't. If every single +human being in America worked for the phone company, +we couldn't match the performance of digital switches: +direct-dialling, three-way calling, speed-calling, call- +waiting, Caller ID, all the rest of the cornucopia +of digital bounty. Replacing computers with operators +is simply not an option any more. + +And yet we still, anachronistically, expect humans to +be running our phone system. It is hard for us +to understand that we have sacrificed huge amounts +of initiative and control to senseless yet powerful machines. +When the phones fail, we want somebody to be responsible. +We want somebody to blame. + +When the Crash of January 15 happened, the American populace +was simply not prepared to understand that enormous landslides +in cyberspace, like the Crash itself, can happen, +and can be nobody's fault in particular. It was easier to believe, +maybe even in some odd way more reassuring to believe, +that some evil person, or evil group, had done this to us. +"Hackers" had done it. With a virus. A trojan horse. +A software bomb. A dirty plot of some kind. People believed this, +responsible people. In 1990, they were looking hard for evidence +to confirm their heartfelt suspicions. + +And they would look in a lot of places. + +Come 1991, however, the outlines of an apparent new reality +would begin to emerge from the fog. + +On July 1 and 2, 1991, computer-software collapses +in telephone switching stations disrupted service in +Washington DC, Pittsburgh, Los Angeles and San Francisco. +Once again, seemingly minor maintenance problems had +crippled the digital System 7. About twelve million +people were affected in the Crash of July 1, 1991. + +Said the New York Times Service: "Telephone company executives +and federal regulators said they were not ruling out the possibility +of sabotage by computer hackers, but most seemed to think the problems +stemmed from some unknown defect in the software running the networks." + +And sure enough, within the week, a red-faced software company, +DSC Communications Corporation of Plano, Texas, owned up +to "glitches" in the "signal transfer point" software that +DSC had designed for Bell Atlantic and Pacific Bell. +The immediate cause of the July 1 Crash was a single +mistyped character: one tiny typographical flaw +in one single line of the software. One mistyped letter, +in one single line, had deprived the nation's capital of phone service. +It was not particularly surprising that this tiny flaw had escaped attention: +a typical System 7 station requires TEN MILLION lines of code. + +On Tuesday, September 17, 1991, came the most spectacular outage yet. +This case had nothing to do with software failures--at least, not directly. +Instead, a group of AT&T's switching stations in New York City had simply +run out of electrical power and shut down cold. Their back-up batteries +had failed. Automatic warning systems were supposed to warn of the loss +of battery power, but those automatic systems had failed as well. + +This time, Kennedy, La Guardia, and Newark airports +all had their voice and data communications cut. +This horrifying event was particularly ironic, as attacks +on airport computers by hackers had long been a standard +nightmare scenario, much trumpeted by computer-security +experts who feared the computer underground. There had even +been a Hollywood thriller about sinister hackers ruining +airport computers--DIE HARD II. + +Now AT&T itself had crippled airports with computer malfunctions-- +not just one airport, but three at once, some of the busiest in the world. + +Air traffic came to a standstill throughout the Greater New York area, +causing more than 500 flights to be cancelled, in a spreading wave +all over America and even into Europe. Another 500 or so flights +were delayed, affecting, all in all, about 85,000 passengers. +(One of these passengers was the chairman of the Federal +Communications Commission.) + +Stranded passengers in New York and New Jersey were further +infuriated to discover that they could not even manage to +make a long distance phone call, to explain their delay +to loved ones or business associates. Thanks to the crash, +about four and a half million domestic calls, and half a million +international calls, failed to get through. + +The September 17 NYC Crash, unlike the previous ones, +involved not a whisper of "hacker" misdeeds. On the contrary, +by 1991, AT&T itself was suffering much of the vilification +that had formerly been directed at hackers. Congressmen were grumbling. +So were state and federal regulators. And so was the press. + +For their part, ancient rival MCI took out snide full-page +newspaper ads in New York, offering their own long-distance +services for the "next time that AT&T goes down." + +"You wouldn't find a classy company like AT&T using such advertising," +protested AT&T Chairman Robert Allen, unconvincingly. Once again, +out came the full-page AT&T apologies in newspapers, apologies for +"an inexcusable culmination of both human and mechanical failure." +(This time, however, AT&T offered no discount on later calls. +Unkind critics suggested that AT&T were worried about setting any precedent +for refunding the financial losses caused by telephone crashes.) + +Industry journals asked publicly if AT&T was "asleep at the switch." +The telephone network, America's purported marvel of high-tech reliability, +had gone down three times in 18 months. Fortune magazine listed the +Crash of September 17 among the "Biggest Business Goofs of 1991," +cruelly parodying AT&T's ad campaign in an article entitled +"AT&T Wants You Back (Safely On the Ground, God Willing)." + +Why had those New York switching systems simply run out of power? +Because no human being had attended to the alarm system. +Why did the alarm systems blare automatically, +without any human being noticing? Because the three +telco technicians who SHOULD have been listening +were absent from their stations in the power-room, +on another floor of the building--attending a training class. +A training class about the alarm systems for the power room! + +"Crashing the System" was no longer "unprecedented" by late 1991. +On the contrary, it no longer even seemed an oddity. By 1991, +it was clear that all the policemen in the world could no longer +"protect" the phone system from crashes. By far the worst crashes +the system had ever had, had been inflicted, by the system, +upon ITSELF. And this time nobody was making cocksure statements +that this was an anomaly, something that would never happen again. +By 1991 the System's defenders had met their nebulous Enemy, +and the Enemy was--the System. + + + +PART TWO: THE DIGITAL UNDERGROUND + + +The date was May 9, 1990. The Pope was touring Mexico City. +Hustlers from the Medellin Cartel were trying to buy +black-market Stinger missiles in Florida. On the comics page, +Doonesbury character Andy was dying of AIDS. And then. . .a highly +unusual item whose novelty and calculated rhetoric won it +headscratching attention in newspapers all over America. + +The US Attorney's office in Phoenix, Arizona, had issued +a press release announcing a nationwide law enforcement crackdown +against "illegal computer hacking activities." The sweep was +officially known as "Operation Sundevil." + +Eight paragraphs in the press release gave the bare facts: +twenty-seven search warrants carried out on May 8, with three arrests, +and a hundred and fifty agents on the prowl in "twelve" cities across America. +(Different counts in local press reports yielded "thirteen," "fourteen," and +"sixteen" cities.) Officials estimated that criminal losses of revenue +to telephone companies "may run into millions of dollars." Credit for +the Sundevil investigations was taken by the US Secret Service, +Assistant US Attorney Tim Holtzen of Phoenix, and the Assistant +Attorney General of Arizona, Gail Thackeray. + +The prepared remarks of Garry M. Jenkins, appearing in a U.S. Department +of Justice press release, were of particular interest. Mr. Jenkins was the +Assistant Director of the US Secret Service, and the highest-ranking federal +official to take any direct public role in the hacker crackdown of 1990. + +"Today, the Secret Service is sending a clear message to those computer hackers +who have decided to violate the laws of this nation in the mistaken belief +that they can successfully avoid detection by hiding behind the relative +anonymity of their computer terminals. (. . .) "Underground groups have been +formed for the purpose of exchanging information relevant to their criminal +activities. These groups often communicate with each other through message +systems between computers called `bulletin boards.' "Our experience shows +that many computer hacker suspects are no longer misguided teenagers, +mischievously playing games with their computers in their bedrooms. +Some are now high tech computer operators using computers to engage +in unlawful conduct." + +Who were these "underground groups" and "high-tech operators?" +Where had they come from? What did they want? Who WERE they? +Were they "mischievous?" Were they dangerous? How had "misguided teenagers" +managed to alarm the United States Secret Service? And just how widespread +was this sort of thing? + +Of all the major players in the Hacker Crackdown: the phone companies, +law enforcement, the civil libertarians, and the "hackers" themselves-- +the "hackers" are by far the most mysterious, by far the hardest to +understand, by far the WEIRDEST. + +Not only are "hackers" novel in their activities, but they come +in a variety of odd subcultures, with a variety of languages, +motives and values. + +The earliest proto-hackers were probably those unsung mischievous +telegraph boys who were summarily fired by the Bell Company in 1878. + +Legitimate "hackers," those computer enthusiasts who are independent-minded +but law-abiding, generally trace their spiritual ancestry to elite technical +universities, especially M.I.T. and Stanford, in the 1960s. + +But the genuine roots of the modern hacker UNDERGROUND can probably be traced +most successfully to a now much-obscured hippie anarchist movement known as +the Yippies. The Yippies, who took their name from the largely fictional +"Youth International Party," carried out a loud and lively policy of surrealistic +subversion and outrageous political mischief. Their basic tenets were flagrant +sexual promiscuity, open and copious drug use, the political overthrow of any +powermonger over thirty years of age, and an immediate end to the war +in Vietnam, by any means necessary, including the psychic levitation +of the Pentagon. + +The two most visible Yippies were Abbie Hoffman and Jerry Rubin. +Rubin eventually became a Wall Street broker. Hoffman, ardently sought +by federal authorities, went into hiding for seven years, +in Mexico, France, and the United States. While on the lam, +Hoffman continued to write and publish, with help from sympathizers +in the American anarcho-leftist underground. Mostly, Hoffman survived +through false ID and odd jobs. Eventually he underwent facial plastic +surgery and adopted an entirely new identity as one "Barry Freed." +After surrendering himself to authorities in 1980, Hoffman spent a year +in prison on a cocaine conviction. + +Hoffman's worldview grew much darker as the glory days of the 1960s faded. +In 1989, he purportedly committed suicide, under odd and, to some, rather +suspicious circumstances. + +Abbie Hoffman is said to have caused the Federal Bureau of Investigation +to amass the single largest investigation file ever opened on an individual +American citizen. (If this is true, it is still questionable whether the +FBI regarded Abbie Hoffman a serious public threat--quite possibly, +his file was enormous simply because Hoffman left colorful legendry +wherever he went). He was a gifted publicist, who regarded electronic +media as both playground and weapon. He actively enjoyed manipulating +network TV and other gullible, image-hungry media, with various weird lies, +mindboggling rumors, impersonation scams, and other sinister distortions, +all absolutely guaranteed to upset cops, Presidential candidates, +and federal judges. Hoffman's most famous work was a book self-reflexively +known as STEAL THIS BOOK, which publicized a number of methods by which young, +penniless hippie agitators might live off the fat of a system supported by +humorless drones. STEAL THIS BOOK, whose title urged readers to damage +the very means of distribution which had put it into their hands, +might be described as a spiritual ancestor of a computer virus. + +Hoffman, like many a later conspirator, made extensive use of +pay-phones for his agitation work--in his case, generally through +the use of cheap brass washers as coin-slugs. + +During the Vietnam War, there was a federal surtax imposed on telephone +service; Hoffman and his cohorts could, and did, argue that in systematically +stealing phone service they were engaging in civil disobedience: +virtuously denying tax funds to an illegal and immoral war. + +But this thin veil of decency was soon dropped entirely. +Ripping-off the System found its own justification in deep alienation +and a basic outlaw contempt for conventional bourgeois values. +Ingenious, vaguely politicized varieties of rip-off, +which might be described as "anarchy by convenience," +became very popular in Yippie circles, and because rip-off +was so useful, it was to survive the Yippie movement itself. + +In the early 1970s, it required fairly limited expertise +and ingenuity to cheat payphones, to divert "free" +electricity and gas service, or to rob vending machines +and parking meters for handy pocket change. It also required +a conspiracy to spread this knowledge, and the gall +and nerve actually to commit petty theft, but the Yippies +had these qualifications in plenty. In June 1971, Abbie +Hoffman and a telephone enthusiast sarcastically known +as "Al Bell" began publishing a newsletter called Youth +International Party Line. This newsletter was dedicated +to collating and spreading Yippie rip-off techniques, +especially of phones, to the joy of the freewheeling +underground and the insensate rage of all straight people. +As a political tactic, phone-service theft ensured +that Yippie advocates would always have ready access +to the long-distance telephone as a medium, despite +the Yippies' chronic lack of organization, discipline, +money, or even a steady home address. + +PARTY LINE was run out of Greenwich Village for a couple of years, +then "Al Bell" more or less defected from the faltering ranks of Yippiedom, +changing the newsletter's name to TAP or Technical Assistance Program. +After the Vietnam War ended, the steam began leaking rapidly out of American +radical dissent. But by this time, "Bell" and his dozen or so +core contributors had the bit between their teeth, +and had begun to derive tremendous gut-level satisfaction +from the sensation of pure TECHNICAL POWER. + +TAP articles, once highly politicized, became pitilessly jargonized +and technical, in homage or parody to the Bell System's own technical +documents, which TAP studied closely, gutted, and reproduced without +permission. The TAP elite revelled in gloating possession +of the specialized knowledge necessary to beat the system. + +"Al Bell" dropped out of the game by the late 70s, +and "Tom Edison" took over; TAP readers (some 1400 of +them, all told) now began to show more interest in telex +switches and the growing phenomenon of computer systems. + +In 1983, "Tom Edison" had his computer stolen and his house +set on fire by an arsonist. This was an eventually mortal blow +to TAP (though the legendary name was to be resurrected +in 1990 by a young Kentuckian computer-outlaw named "Predat0r.") + +# + +Ever since telephones began to make money, there have been +people willing to rob and defraud phone companies. +The legions of petty phone thieves vastly outnumber those +"phone phreaks" who "explore the system" for the sake +of the intellectual challenge. The New York metropolitan area +(long in the vanguard of American crime) claims over 150,000 +physical attacks on pay telephones every year! Studied carefully, +a modern payphone reveals itself as a little fortress, carefully +designed and redesigned over generations, to resist coin-slugs, +zaps of electricity, chunks of coin-shaped ice, prybars, magnets, +lockpicks, blasting caps. Public pay- phones must survive in a world +of unfriendly, greedy people, and a modern payphone is as exquisitely +evolved as a cactus. +Because the phone network pre-dates the computer network, +the scofflaws known as "phone phreaks" pre-date the scofflaws +known as "computer hackers." In practice, today, the line +between "phreaking" and "hacking" is very blurred, +just as the distinction between telephones and computers +has blurred. The phone system has been digitized, +and computers have learned to "talk" over phone-lines. +What's worse--and this was the point of the Mr. Jenkins +of the Secret Service--some hackers have learned to steal, +and some thieves have learned to hack. + +Despite the blurring, one can still draw a few useful +behavioral distinctions between "phreaks" and "hackers." +Hackers are intensely interested in the "system" per se, +and enjoy relating to machines. "Phreaks" are more +social, manipulating the system in a rough-and-ready +fashion in order to get through to other human beings, +fast, cheap and under the table. + +Phone phreaks love nothing so much as "bridges," +illegal conference calls of ten or twelve chatting +conspirators, seaboard to seaboard, lasting for many hours +--and running, of course, on somebody else's tab, +preferably a large corporation's. + +As phone-phreak conferences wear on, people drop out +(or simply leave the phone off the hook, while they +sashay off to work or school or babysitting), +and new people are phoned up and invited to join in, +from some other continent, if possible. Technical trivia, +boasts, brags, lies, head-trip deceptions, weird rumors, +and cruel gossip are all freely exchanged. + +The lowest rung of phone-phreaking is the theft of telephone access codes. +Charging a phone call to somebody else's stolen number is, of course, +a pig-easy way of stealing phone service, requiring practically no +technical expertise. This practice has been very widespread, +especially among lonely people without much money who are far from home. +Code theft has flourished especially in college dorms, military bases, +and, notoriously, among roadies for rock bands. Of late, code theft +has spread very rapidly among Third Worlders in the US, who pile up +enormous unpaid long-distance bills to the Caribbean, South America, +and Pakistan. + +The simplest way to steal phone-codes is simply to look over +a victim's shoulder as he punches-in his own code-number +on a public payphone. This technique is known as "shoulder-surfing," +and is especially common in airports, bus terminals, and train stations. +The code is then sold by the thief for a few dollars. The buyer abusing +the code has no computer expertise, but calls his Mom in New York, +Kingston or Caracas and runs up a huge bill with impunity. The losses +from this primitive phreaking activity are far, far greater than the +monetary losses caused by computer-intruding hackers. + +In the mid-to-late 1980s, until the introduction of sterner telco +security measures, COMPUTERIZED code theft worked like a charm, +and was virtually omnipresent throughout the digital underground, +among phreaks and hackers alike. This was accomplished through +programming one's computer to try random code numbers over the telephone +until one of them worked. Simple programs to do this were widely available +in the underground; a computer running all night was likely to come up with +a dozen or so useful hits. This could be repeated week after week until +one had a large library of stolen codes. + +Nowadays, the computerized dialling of hundreds of numbers +can be detected within hours and swiftly traced. +If a stolen code is repeatedly abused, this too can +be detected within a few hours. But for years in the 1980s, +the publication of stolen codes was a kind of elementary etiquette +for fledgling hackers. The simplest way to establish your bona-fides +as a raider was to steal a code through repeated random dialling +and offer it to the "community" for use. Codes could be both stolen, +and used, simply and easily from the safety of one's own bedroom, +with very little fear of detection or punishment. + +Before computers and their phone-line modems entered American homes +in gigantic numbers, phone phreaks had their own special telecommunications +hardware gadget, the famous "blue box." This fraud device (now rendered +increasingly useless by the digital evolution of the phone system) could +trick switching systems into granting free access to long-distance lines. +It did this by mimicking the system's own signal, a tone of 2600 hertz. + +Steven Jobs and Steve Wozniak, the founders of Apple Computer, Inc., +once dabbled in selling blue-boxes in college dorms in California. +For many, in the early days of phreaking, blue-boxing was scarcely +perceived as "theft," but rather as a fun (if sneaky) way to use +excess phone capacity harmlessly. After all, the long-distance +lines were JUST SITTING THERE. . . . Whom did it hurt, really? +If you're not DAMAGING the system, and you're not USING UP ANY +TANGIBLE RESOURCE, and if nobody FINDS OUT what you did, +then what real harm have you done? What exactly HAVE you "stolen," +anyway? If a tree falls in the forest and nobody hears it, +how much is the noise worth? Even now this remains a rather +dicey question. + +Blue-boxing was no joke to the phone companies, however. +Indeed, when Ramparts magazine, a radical publication in California, +printed the wiring schematics necessary to create a mute box in June 1972, +the magazine was seized by police and Pacific Bell phone-company officials. +The mute box, a blue-box variant, allowed its user to receive long-distance +calls free of charge to the caller. This device was closely described in a +Ramparts article wryly titled "Regulating the Phone Company In Your Home." +Publication of this article was held to be in violation of Californian +State Penal Code section 502.7, which outlaws ownership of wire-fraud +devices and the selling of "plans or instructions for any instrument, +apparatus, or device intended to avoid telephone toll charges." + +Issues of Ramparts were recalled or seized on the newsstands, +and the resultant loss of income helped put the magazine out of business. +This was an ominous precedent for free-expression issues, but the telco's +crushing of a radical-fringe magazine passed without serious challenge +at the time. Even in the freewheeling California 1970s, it was widely felt +that there was something sacrosanct about what the phone company knew; +that the telco had a legal and moral right to protect itself by shutting +off the flow of such illicit information. Most telco information was so +"specialized" that it would scarcely be understood by any honest member +of the public. If not published, it would not be missed. To print such +material did not seem part of the legitimate role of a free press. + +In 1990 there would be a similar telco-inspired attack +on the electronic phreak/hacking "magazine" Phrack. +The Phrack legal case became a central issue in the +Hacker Crackdown, and gave rise to great controversy. +Phrack would also be shut down, for a time, at least, +but this time both the telcos and their law-enforcement +allies would pay a much larger price for their actions. +The Phrack case will be examined in detail, later. + +Phone-phreaking as a social practice is still very +much alive at this moment. Today, phone-phreaking +is thriving much more vigorously than the better-known +and worse-feared practice of "computer hacking." +New forms of phreaking are spreading rapidly, following +new vulnerabilities in sophisticated phone services. + +Cellular phones are especially vulnerable; their chips +can be re-programmed to present a false caller ID +and avoid billing. Doing so also avoids police tapping, +making cellular-phone abuse a favorite among drug-dealers. +"Call-sell operations" using pirate cellular phones can, +and have, been run right out of the backs of cars, which move +from "cell" to "cell" in the local phone system, retailing +stolen long-distance service, like some kind of demented +electronic version of the neighborhood ice-cream truck. + +Private branch-exchange phone systems in large corporations +can be penetrated; phreaks dial-up a local company, enter its +internal phone-system, hack it, then use the company's own +PBX system to dial back out over the public network, +causing the company to be stuck with the resulting +long-distance bill. This technique is known as "diverting." +"Diverting" can be very costly, especially because phreaks +tend to travel in packs and never stop talking. +Perhaps the worst by-product of this "PBX fraud" +is that victim companies and telcos have sued one another +over the financial responsibility for the stolen calls, +thus enriching not only shabby phreaks but well-paid lawyers. + +"Voice-mail systems" can also be abused; phreaks +can seize their own sections of these sophisticated +electronic answering machines, and use them for trading +codes or knowledge of illegal techniques. Voice-mail +abuse does not hurt the company directly, but finding +supposedly empty slots in your company's answering +machine all crammed with phreaks eagerly chattering +and hey-duding one another in impenetrable jargon can +cause sensations of almost mystical repulsion and dread. + +Worse yet, phreaks have sometimes been known to react +truculently to attempts to "clean up" the voice-mail system. +Rather than humbly acquiescing to being thrown out of their playground, +they may very well call up the company officials at work (or at home) +and loudly demand free voice-mail addresses of their very own. +Such bullying is taken very seriously by spooked victims. + +Acts of phreak revenge against straight people are rare, +but voice-mail systems are especially tempting and vulnerable, +and an infestation of angry phreaks in one's voice-mail system is no joke. +They can erase legitimate messages; or spy on private messages; +or harass users with recorded taunts and obscenities. +They've even been known to seize control of voice-mail security, +and lock out legitimate users, or even shut down the system entirely. + +Cellular phone-calls, cordless phones, and ship-to-shore +telephony can all be monitored by various forms of radio; +this kind of "passive monitoring" is spreading explosively today. +Technically eavesdropping on other people's cordless and cellular +phone-calls is the fastest-growing area in phreaking today. +This practice strongly appeals to the lust for power and conveys +gratifying sensations of technical superiority over the eavesdropping +victim. Monitoring is rife with all manner of tempting evil mischief. +Simple prurient snooping is by far the most common activity. +But credit-card numbers unwarily spoken over the phone can be recorded, +stolen and used. And tapping people's phone-calls (whether through +active telephone taps or passive radio monitors) does lend itself +conveniently to activities like blackmail, industrial espionage, +and political dirty tricks. + +It should be repeated that telecommunications fraud, +the theft of phone service, causes vastly greater monetary +losses than the practice of entering into computers by stealth. +Hackers are mostly young suburban American white males, +and exist in their hundreds--but "phreaks" come from both sexes +and from many nationalities, ages and ethnic backgrounds, +and are flourishing in the thousands. + +# + +The term "hacker" has had an unfortunate history. +This book, The Hacker Crackdown, has little to say about +"hacking" in its finer, original sense. The term can signify +the free-wheeling intellectual exploration of the highest +and deepest potential of computer systems. Hacking can +describe the determination to make access to computers +and information as free and open as possible. Hacking +can involve the heartfelt conviction that beauty can +be found in computers, that the fine aesthetic in a perfect +program can liberate the mind and spirit. This is "hacking" +as it was defined in Steven Levy's much-praised history +of the pioneer computer milieu, Hackers, published in 1984. + +Hackers of all kinds are absolutely soaked through with heroic +anti-bureaucratic sentiment. Hackers long for recognition +as a praiseworthy cultural archetype, the postmodern electronic +equivalent of the cowboy and mountain man. Whether they deserve +such a reputation is something for history to decide. But many hackers-- +including those outlaw hackers who are computer intruders, and whose +activities are defined as criminal--actually attempt to LIVE UP TO +this techno-cowboy reputation. And given that electronics and +telecommunications are still largely unexplored territories, +there is simply NO TELLING what hackers might uncover. + +For some people, this freedom is the very breath of oxygen, +the inventive spontaneity that makes life worth living +and that flings open doors to marvellous possibility and +individual empowerment. But for many people +--and increasingly so--the hacker is an ominous figure, +a smart-aleck sociopath ready to burst out of his basement +wilderness and savage other people's lives for his own +anarchical convenience. + +Any form of power without responsibility, without direct +and formal checks and balances, is frightening to people-- +and reasonably so. It should be frankly admitted that +hackers ARE frightening, and that the basis of this fear +is not irrational. + +Fear of hackers goes well beyond the fear of merely criminal activity. + +Subversion and manipulation of the phone system +is an act with disturbing political overtones. +In America, computers and telephones are potent symbols +of organized authority and the technocratic business elite. + +But there is an element in American culture that +has always strongly rebelled against these symbols; +rebelled against all large industrial computers +and all phone companies. A certain anarchical tinge deep +in the American soul delights in causing confusion and pain +to all bureaucracies, including technological ones. + +There is sometimes malice and vandalism in this attitude, +but it is a deep and cherished part of the American national character. +The outlaw, the rebel, the rugged individual, the pioneer, +the sturdy Jeffersonian yeoman, the private citizen resisting +interference in his pursuit of happiness--these are figures that all +Americans recognize, and that many will strongly applaud and defend. + +Many scrupulously law-abiding citizens today do cutting-edge work +with electronics--work that has already had tremendous social influence +and will have much more in years to come. In all truth, these talented, +hardworking, law-abiding, mature, adult people are far more disturbing +to the peace and order of the current status quo than any scofflaw group +of romantic teenage punk kids. These law-abiding hackers have the power, +ability, and willingness to influence other people's lives quite unpredictably. +They have means, motive, and opportunity to meddle drastically with the +American social order. When corralled into governments, universities, +or large multinational companies, and forced to follow rulebooks +and wear suits and ties, they at least have some conventional halters +on their freedom of action. But when loosed alone, or in small groups, +and fired by imagination and the entrepreneurial spirit, they can move +mountains--causing landslides that will likely crash directly into your +office and living room. + +These people, as a class, instinctively recognize that a public, +politicized attack on hackers will eventually spread to them-- +that the term "hacker," once demonized, might be used to knock +their hands off the levers of power and choke them out of existence. +There are hackers today who fiercely and publicly resist any besmirching +of the noble title of hacker. Naturally and understandably, they deeply +resent the attack on their values implicit in using the word "hacker" +as a synonym for computer-criminal. + +This book, sadly but in my opinion unavoidably, rather adds +to the degradation of the term. It concerns itself mostly with "hacking" +in its commonest latter-day definition, i.e., intruding into computer +systems by stealth and without permission. The term "hacking" is used +routinely today by almost all law enforcement officials with any +professional interest in computer fraud and abuse. American police +describe almost any crime committed with, by, through, or against +a computer as hacking. + +Most importantly, "hacker" is what computer-intruders +choose to call THEMSELVES. Nobody who "hacks" into systems +willingly describes himself (rarely, herself) as a "computer intruder," +"computer trespasser," "cracker," "wormer," "darkside hacker" +or "high tech street gangster." Several other demeaning terms +have been invented in the hope that the press and public +will leave the original sense of the word alone. But few people +actually use these terms. (I exempt the term "cyberpunk," +which a few hackers and law enforcement people actually do use. +The term "cyberpunk" is drawn from literary criticism and has +some odd and unlikely resonances, but, like hacker, +cyberpunk too has become a criminal pejorative today.) + +In any case, breaking into computer systems was hardly alien +to the original hacker tradition. The first tottering systems +of the 1960s required fairly extensive internal surgery merely +to function day-by-day. Their users "invaded" the deepest, +most arcane recesses of their operating software almost +as a matter of routine. "Computer security" in these early, +primitive systems was at best an afterthought. What security +there was, was entirely physical, for it was assumed that +anyone allowed near this expensive, arcane hardware would be +a fully qualified professional expert. + +In a campus environment, though, this meant that grad students, +teaching assistants, undergraduates, and eventually, +all manner of dropouts and hangers-on ended up accessing +and often running the works. + +Universities, even modern universities, are not in +the business of maintaining security over information. +On the contrary, universities, as institutions, pre-date +the "information economy" by many centuries and are not- +for-profit cultural entities, whose reason for existence +(purportedly) is to discover truth, codify it through +techniques of scholarship, and then teach it. Universities +are meant to PASS THE TORCH OF CIVILIZATION, not just +download data into student skulls, and the values of the +academic community are strongly at odds with those of all +would-be information empires. Teachers at all levels, from +kindergarten up, have proven to be shameless and persistent +software and data pirates. Universities do not merely +"leak information" but vigorously broadcast free thought. + +This clash of values has been fraught with controversy. +Many hackers of the 1960s remember their professional +apprenticeship as a long guerilla war against the uptight +mainframe-computer "information priesthood." These computer-hungry +youngsters had to struggle hard for access to computing power, +and many of them were not above certain, er, shortcuts. +But, over the years, this practice freed computing +from the sterile reserve of lab-coated technocrats and +was largely responsible for the explosive growth of computing +in general society--especially PERSONAL computing. + +Access to technical power acted like catnip on certain +of these youngsters. Most of the basic techniques of +computer intrusion: password cracking, trapdoors, backdoors, +trojan horses--were invented in college environments in the 1960s, +in the early days of network computing. Some off-the-cuff +experience at computer intrusion was to be in the informal +resume of most "hackers" and many future industry giants. +Outside of the tiny cult of computer enthusiasts, few people +thought much about the implications of "breaking into" +computers. This sort of activity had not yet been publicized, +much less criminalized. + +In the 1960s, definitions of "property" and "privacy" +had not yet been extended to cyberspace. Computers +were not yet indispensable to society. There were no vast +databanks of vulnerable, proprietary information stored +in computers, which might be accessed, copied without +permission, erased, altered, or sabotaged. The stakes +were low in the early days--but they grew every year, +exponentially, as computers themselves grew. + +By the 1990s, commercial and political pressures +had become overwhelming, and they broke the social +boundaries of the hacking subculture. Hacking +had become too important to be left to the hackers. +Society was now forced to tackle the intangible nature +of cyberspace-as-property, cyberspace as privately-owned +unreal-estate. In the new, severe, responsible, high-stakes +context of the "Information Society" of the 1990s, +"hacking" was called into question. + +What did it mean to break into a computer without +permission and use its computational power, or look +around inside its files without hurting anything? +What were computer-intruding hackers, anyway--how should +society, and the law, best define their actions? +Were they just BROWSERS, harmless intellectual explorers? +Were they VOYEURS, snoops, invaders of privacy? Should +they be sternly treated as potential AGENTS OF ESPIONAGE, +or perhaps as INDUSTRIAL SPIES? Or were they best +defined as TRESPASSERS, a very common teenage +misdemeanor? Was hacking THEFT OF SERVICE? +(After all, intruders were getting someone else's +computer to carry out their orders, without permission +and without paying). Was hacking FRAUD? Maybe it was +best described as IMPERSONATION. The commonest mode +of computer intrusion was (and is) to swipe or snoop +somebody else's password, and then enter the computer +in the guise of another person--who is commonly stuck +with the blame and the bills. + +Perhaps a medical metaphor was better--hackers should +be defined as "sick," as COMPUTER ADDICTS unable +to control their irresponsible, compulsive behavior. + +But these weighty assessments meant little to the +people who were actually being judged. From inside +the underground world of hacking itself, all these +perceptions seem quaint, wrongheaded, stupid, or meaningless. +The most important self-perception of underground hackers-- +from the 1960s, right through to the present day--is that +they are an ELITE. The day-to-day struggle in the underground +is not over sociological definitions--who cares?--but for power, +knowledge, and status among one's peers. + +When you are a hacker, it is your own inner conviction +of your elite status that enables you to break, or let +us say "transcend," the rules. It is not that ALL rules +go by the board. The rules habitually broken by hackers +are UNIMPORTANT rules--the rules of dopey greedhead telco +bureaucrats and pig-ignorant government pests. + +Hackers have their OWN rules, which separate behavior +which is cool and elite, from behavior which is rodentlike, +stupid and losing. These "rules," however, are mostly unwritten +and enforced by peer pressure and tribal feeling. Like all rules +that depend on the unspoken conviction that everybody else +is a good old boy, these rules are ripe for abuse. The mechanisms +of hacker peer- pressure, "teletrials" and ostracism, are rarely used +and rarely work. Back-stabbing slander, threats, and electronic +harassment are also freely employed in down-and-dirty intrahacker feuds, +but this rarely forces a rival out of the scene entirely. The only real +solution for the problem of an utterly losing, treacherous and rodentlike +hacker is to TURN HIM IN TO THE POLICE. Unlike the Mafia or Medellin Cartel, +the hacker elite cannot simply execute the bigmouths, creeps and troublemakers +among their ranks, so they turn one another in with astonishing frequency. + +There is no tradition of silence or OMERTA in the hacker underworld. +Hackers can be shy, even reclusive, but when they do talk, hackers +tend to brag, boast and strut. Almost everything hackers do is INVISIBLE; +if they don't brag, boast, and strut about it, then NOBODY WILL EVER KNOW. +If you don't have something to brag, boast, and strut about, then nobody +in the underground will recognize you and favor you with vital cooperation +and respect. + +The way to win a solid reputation in the underground +is by telling other hackers things that could only +have been learned by exceptional cunning and stealth. +Forbidden knowledge, therefore, is the basic currency +of the digital underground, like seashells among +Trobriand Islanders. Hackers hoard this knowledge, +and dwell upon it obsessively, and refine it, +and bargain with it, and talk and talk about it. + +Many hackers even suffer from a strange obsession to TEACH-- +to spread the ethos and the knowledge of the digital underground. +They'll do this even when it gains them no particular advantage +and presents a grave personal risk. + +And when that risk catches up with them, they will go right on teaching +and preaching--to a new audience this time, their interrogators from law +enforcement. Almost every hacker arrested tells everything he knows-- +all about his friends, his mentors, his disciples--legends, threats, +horror stories, dire rumors, gossip, hallucinations. This is, of course, +convenient for law enforcement--except when law enforcement begins +to believe hacker legendry. + +Phone phreaks are unique among criminals in their willingness +to call up law enforcement officials--in the office, at their homes-- +and give them an extended piece of their mind. It is hard not to +interpret this as BEGGING FOR ARREST, and in fact it is an act +of incredible foolhardiness. Police are naturally nettled +by these acts of chutzpah and will go well out of their way +to bust these flaunting idiots. But it can also be interpreted +as a product of a world-view so elitist, so closed and hermetic, +that electronic police are simply not perceived as "police," +but rather as ENEMY PHONE PHREAKS who should be scolded +into behaving "decently." + +Hackers at their most grandiloquent perceive themselves +as the elite pioneers of a new electronic world. +Attempts to make them obey the democratically +established laws of contemporary American society are +seen as repression and persecution. After all, they argue, +if Alexander Graham Bell had gone along with the rules +of the Western Union telegraph company, there would have +been no telephones. If Jobs and Wozniak had believed +that IBM was the be-all and end-all, there would have +been no personal computers. If Benjamin Franklin and +Thomas Jefferson had tried to "work within the system" +there would have been no United States. + +Not only do hackers privately believe this as an article of faith, +but they have been known to write ardent manifestos about it. +Here are some revealing excerpts from an especially vivid hacker manifesto: +"The Techno-Revolution" by "Dr. Crash," which appeared in electronic +form in Phrack Volume 1, Issue 6, Phile 3. + + +"To fully explain the true motives behind hacking, +we must first take a quick look into the past. In the 1960s, +a group of MIT students built the first modern computer system. +This wild, rebellious group of young men were the first to bear +the name `hackers.' The systems that they developed were intended +to be used to solve world problems and to benefit all of mankind. +"As we can see, this has not been the case. The computer system +has been solely in the hands of big businesses and the government. +The wonderful device meant to enrich life has become a weapon which +dehumanizes people. To the government and large businesses, +people are no more than disk space, and the government doesn't +use computers to arrange aid for the poor, but to control nuclear +death weapons. The average American can only have access +to a small microcomputer which is worth only a fraction +of what they pay for it. The businesses keep the +true state-of-the-art equipment away from the people +behind a steel wall of incredibly high prices and bureaucracy. +It is because of this state of affairs that hacking was born. (. . .) +"Of course, the government doesn't want the monopoly of technology broken, +so they have outlawed hacking and arrest anyone who is caught. (. . .) +The phone company is another example of technology abused and kept +from people with high prices. (. . .) "Hackers often find that their +existing equipment, due to the monopoly tactics of computer companies, +is inefficient for their purposes. Due to the exorbitantly high prices, +it is impossible to legally purchase the necessary equipment. +This need has given still another segment of the fight: Credit Carding. +Carding is a way of obtaining the necessary goods without paying for them. +It is again due to the companies' stupidity that Carding is so easy, +and shows that the world's businesses are in the hands of those +with considerably less technical know-how than we, the hackers. (. . .) +"Hacking must continue. We must train newcomers to the art of hacking. +(. . . .) And whatever you do, continue the fight. Whether you know it +or not, if you are a hacker, you are a revolutionary. Don't worry, +you're on the right side." + +The defense of "carding" is rare. Most hackers regard credit-card +theft as "poison" to the underground, a sleazy and immoral effort that, +worse yet, is hard to get away with. Nevertheless, manifestos advocating +credit-card theft, the deliberate crashing of computer systems, +and even acts of violent physical destruction such as vandalism +and arson do exist in the underground. These boasts and threats +are taken quite seriously by the police. And not every hacker +is an abstract, Platonic computer-nerd. Some few are quite experienced +at picking locks, robbing phone-trucks, and breaking and entering buildings. + +Hackers vary in their degree of hatred for authority +and the violence of their rhetoric. But, at a bottom line, +they are scofflaws. They don't regard the current rules +of electronic behavior as respectable efforts to preserve +law and order and protect public safety. They regard these +laws as immoral efforts by soulless corporations to protect +their profit margins and to crush dissidents. "Stupid" people, +including police, businessmen, politicians, and journalists, +simply have no right to judge the actions of those possessed of genius, +techno-revolutionary intentions, and technical expertise. + +# + +Hackers are generally teenagers and college kids not +engaged in earning a living. They often come from fairly +well-to-do middle-class backgrounds, and are markedly +anti-materialistic (except, that is, when it comes to +computer equipment). Anyone motivated by greed for +mere money (as opposed to the greed for power, +knowledge and status) is swiftly written-off as a narrow- +minded breadhead whose interests can only be corrupt +and contemptible. Having grown up in the 1970s and +1980s, the young Bohemians of the digital underground +regard straight society as awash in plutocratic corruption, +where everyone from the President down is for sale and +whoever has the gold makes the rules. + +Interestingly, there's a funhouse-mirror image of this attitude +on the other side of the conflict. The police are also +one of the most markedly anti-materialistic groups +in American society, motivated not by mere money +but by ideals of service, justice, esprit-de-corps, +and, of course, their own brand of specialized knowledge +and power. Remarkably, the propaganda war between cops +and hackers has always involved angry allegations +that the other side is trying to make a sleazy buck. +Hackers consistently sneer that anti-phreak prosecutors +are angling for cushy jobs as telco lawyers and that +computer-crime police are aiming to cash in later +as well-paid computer-security consultants in the private sector. + +For their part, police publicly conflate all +hacking crimes with robbing payphones with crowbars. +Allegations of "monetary losses" from computer intrusion +are notoriously inflated. The act of illicitly copying +a document from a computer is morally equated with +directly robbing a company of, say, half a million dollars. +The teenage computer intruder in possession of this "proprietary" +document has certainly not sold it for such a sum, would likely +have little idea how to sell it at all, and quite probably +doesn't even understand what he has. He has not made a cent +in profit from his felony but is still morally equated with +a thief who has robbed the church poorbox and lit out for Brazil. + +Police want to believe that all hackers are thieves. +It is a tortuous and almost unbearable act for the American +justice system to put people in jail because they want +to learn things which are forbidden for them to know. +In an American context, almost any pretext for punishment +is better than jailing people to protect certain restricted +kinds of information. Nevertheless, POLICING INFORMATION +is part and parcel of the struggle against hackers. + +This dilemma is well exemplified by the remarkable +activities of "Emmanuel Goldstein," editor and publisher +of a print magazine known as 2600: The Hacker Quarterly. +Goldstein was an English major at Long Island's State University +of New York in the '70s, when he became involved with the local +college radio station. His growing interest in electronics +caused him to drift into Yippie TAP circles and thus into +the digital underground, where he became a self-described +techno-rat. His magazine publishes techniques of computer +intrusion and telephone "exploration" as well as gloating +exposes of telco misdeeds and governmental failings. + +Goldstein lives quietly and very privately in a large, +crumbling Victorian mansion in Setauket, New York. +The seaside house is decorated with telco decals, chunks of +driftwood, and the basic bric-a-brac of a hippie crash-pad. +He is unmarried, mildly unkempt, and survives mostly +on TV dinners and turkey-stuffing eaten straight out +of the bag. Goldstein is a man of considerable charm +and fluency, with a brief, disarming smile and the kind +of pitiless, stubborn, thoroughly recidivist integrity +that America's electronic police find genuinely alarming. + +Goldstein took his nom-de-plume, or "handle," from +a character in Orwell's 1984, which may be taken, +correctly, as a symptom of the gravity of his sociopolitical +worldview. He is not himself a practicing computer +intruder, though he vigorously abets these actions, +especially when they are pursued against large +corporations or governmental agencies. Nor is he a thief, +for he loudly scorns mere theft of phone service, in favor +of "exploring and manipulating the system." He is probably +best described and understood as a DISSIDENT. + +Weirdly, Goldstein is living in modern America +under conditions very similar to those of former +East European intellectual dissidents. In other words, +he flagrantly espouses a value-system that is deeply +and irrevocably opposed to the system of those in power +and the police. The values in 2600 are generally expressed +in terms that are ironic, sarcastic, paradoxical, or just +downright confused. But there's no mistaking their +radically anti-authoritarian tenor. 2600 holds that +technical power and specialized knowledge, of any kind +obtainable, belong by right in the hands of those individuals +brave and bold enough to discover them--by whatever means necessary. +Devices, laws, or systems that forbid access, and the free +spread of knowledge, are provocations that any free +and self-respecting hacker should relentlessly attack. +The "privacy" of governments, corporations and other soulless +technocratic organizations should never be protected +at the expense of the liberty and free initiative +of the individual techno-rat. + +However, in our contemporary workaday world, both governments +and corporations are very anxious indeed to police information +which is secret, proprietary, restricted, confidential, +copyrighted, patented, hazardous, illegal, unethical, +embarrassing, or otherwise sensitive. This makes Goldstein +persona non grata, and his philosophy a threat. + +Very little about the conditions of Goldstein's daily +life would astonish, say, Vaclav Havel. (We may note +in passing that President Havel once had his word-processor +confiscated by the Czechoslovak police.) Goldstein lives +by SAMIZDAT, acting semi-openly as a data-center +for the underground, while challenging the powers-that-be +to abide by their own stated rules: freedom of speech +and the First Amendment. + +Goldstein thoroughly looks and acts the part of techno-rat, +with shoulder-length ringlets and a piratical black +fisherman's-cap set at a rakish angle. He often shows up +like Banquo's ghost at meetings of computer professionals, +where he listens quietly, half-smiling and taking thorough notes. + +Computer professionals generally meet publicly, +and find it very difficult to rid themselves of Goldstein +and his ilk without extralegal and unconstitutional actions. +Sympathizers, many of them quite respectable people +with responsible jobs, admire Goldstein's attitude and +surreptitiously pass him information. An unknown but +presumably large proportion of Goldstein's 2,000-plus +readership are telco security personnel and police, +who are forced to subscribe to 2600 to stay abreast +of new developments in hacking. They thus find themselves +PAYING THIS GUY'S RENT while grinding their teeth in anguish, +a situation that would have delighted Abbie Hoffman +(one of Goldstein's few idols). + +Goldstein is probably the best-known public representative +of the hacker underground today, and certainly the best-hated. +Police regard him as a Fagin, a corrupter of youth, and speak +of him with untempered loathing. He is quite an accomplished gadfly. +After the Martin Luther King Day Crash of 1990, Goldstein, +for instance, adeptly rubbed salt into the wound in the pages of 2600. +"Yeah, it was fun for the phone phreaks as we watched the network crumble," +he admitted cheerfully. "But it was also an ominous sign of what's +to come. . . . Some AT&T people, aided by well-meaning but ignorant media, +were spreading the notion that many companies had the same software +and therefore could face the same problem someday. Wrong. This was +entirely an AT&T software deficiency. Of course, other companies could +face entirely DIFFERENT software problems. But then, so too could AT&T." + +After a technical discussion of the system's failings, +the Long Island techno-rat went on to offer thoughtful +criticism to the gigantic multinational's hundreds of +professionally qualified engineers. "What we don't know +is how a major force in communications like AT&T could +be so sloppy. What happened to backups? Sure, +computer systems go down all the time, but people +making phone calls are not the same as people logging +on to computers. We must make that distinction. It's not +acceptable for the phone system or any other essential +service to `go down.' If we continue to trust technology +without understanding it, we can look forward to many +variations on this theme. + +"AT&T owes it to its customers to be prepared to INSTANTLY +switch to another network if something strange and unpredictable +starts occurring. The news here isn't so much the failure +of a computer program, but the failure of AT&T's entire structure." + +The very idea of this. . . . this PERSON. . . . offering +"advice" about "AT&T's entire structure" is more than +some people can easily bear. How dare this near-criminal +dictate what is or isn't "acceptable" behavior from AT&T? +Especially when he's publishing, in the very same issue, +detailed schematic diagrams for creating various switching-network +signalling tones unavailable to the public. + +"See what happens when you drop a `silver box' tone or two +down your local exchange or through different long distance +service carriers," advises 2600 contributor "Mr. Upsetter" +in "How To Build a Signal Box." "If you experiment systematically +and keep good records, you will surely discover something interesting." + +This is, of course, the scientific method, generally regarded +as a praiseworthy activity and one of the flowers of modern civilization. +One can indeed learn a great deal with this sort of structured +intellectual activity. Telco employees regard this mode of "exploration" +as akin to flinging sticks of dynamite into their pond to see what lives +on the bottom. + +2600 has been published consistently since 1984. +It has also run a bulletin board computer system, +printed 2600 T-shirts, taken fax calls. . . . +The Spring 1991 issue has an interesting announcement on page 45: +"We just discovered an extra set of wires attached to our fax line +and heading up the pole. (They've since been clipped.) +Your faxes to us and to anyone else could be monitored." +In the worldview of 2600, the tiny band of techno-rat brothers +(rarely, sisters) are a beseiged vanguard of the truly free and honest. +The rest of the world is a maelstrom of corporate crime and high-level +governmental corruption, occasionally tempered with well-meaning +ignorance. To read a few issues in a row is to enter a nightmare +akin to Solzhenitsyn's, somewhat tempered by the fact that 2600 +is often extremely funny. + +Goldstein did not become a target of the Hacker Crackdown, +though he protested loudly, eloquently, and publicly about it, +and it added considerably to his fame. It was not that he is not +regarded as dangerous, because he is so regarded. Goldstein has had +brushes with the law in the past: in 1985, a 2600 bulletin board +computer was seized by the FBI, and some software on it was formally +declared "a burglary tool in the form of a computer program." +But Goldstein escaped direct repression in 1990, because his +magazine is printed on paper, and recognized as subject +to Constitutional freedom of the press protection. +As was seen in the Ramparts case, this is far from +an absolute guarantee. Still, as a practical matter, +shutting down 2600 by court-order would create so much +legal hassle that it is simply unfeasible, at least +for the present. Throughout 1990, both Goldstein +and his magazine were peevishly thriving. + +Instead, the Crackdown of 1990 would concern itself +with the computerized version of forbidden data. +The crackdown itself, first and foremost, was about +BULLETIN BOARD SYSTEMS. Bulletin Board Systems, most often +known by the ugly and un-pluralizable acronym "BBS," are +the life-blood of the digital underground. Boards were +also central to law enforcement's tactics and strategy +in the Hacker Crackdown. + +A "bulletin board system" can be formally defined as +a computer which serves as an information and message- +passing center for users dialing-up over the phone-lines +through the use of modems. A "modem," or modulator- +demodulator, is a device which translates the digital +impulses of computers into audible analog telephone +signals, and vice versa. Modems connect computers +to phones and thus to each other. + +Large-scale mainframe computers have been connected since the 1960s, +but PERSONAL computers, run by individuals out of their homes, +were first networked in the late 1970s. The "board" created +by Ward Christensen and Randy Suess in February 1978, +in Chicago, Illinois, is generally regarded as the first +personal-computer bulletin board system worthy of the name. + +Boards run on many different machines, employing many +different kinds of software. Early boards were crude and buggy, +and their managers, known as "system operators" or "sysops," +were hard-working technical experts who wrote their own software. +But like most everything else in the world of electronics, +boards became faster, cheaper, better-designed, and generally +far more sophisticated throughout the 1980s. They also moved +swiftly out of the hands of pioneers and into those of the +general public. By 1985 there were something in the +neighborhood of 4,000 boards in America. By 1990 it was +calculated, vaguely, that there were about 30,000 boards in +the US, with uncounted thousands overseas. + +Computer bulletin boards are unregulated enterprises. +Running a board is a rough-and-ready, catch-as-catch-can proposition. +Basically, anybody with a computer, modem, software and a phone-line +can start a board. With second-hand equipment and public-domain +free software, the price of a board might be quite small-- +less than it would take to publish a magazine or even a +decent pamphlet. Entrepreneurs eagerly sell bulletin-board +software, and will coach nontechnical amateur sysops in its use. + +Boards are not "presses." They are not magazines, +or libraries, or phones, or CB radios, or traditional cork +bulletin boards down at the local laundry, though they +have some passing resemblance to those earlier media. +Boards are a new medium--they may even be a LARGE NUMBER of new media. + +Consider these unique characteristics: boards are cheap, +yet they can have a national, even global reach. +Boards can be contacted from anywhere in the global +telephone network, at NO COST to the person running the board-- +the caller pays the phone bill, and if the caller is local, +the call is free. Boards do not involve an editorial elite +addressing a mass audience. The "sysop" of a board is not +an exclusive publisher or writer--he is managing an electronic salon, +where individuals can address the general public, play the part +of the general public, and also exchange private mail +with other individuals. And the "conversation" on boards, +though fluid, rapid, and highly interactive, is not spoken, +but written. It is also relatively anonymous, sometimes completely so. + +And because boards are cheap and ubiquitous, regulations +and licensing requirements would likely be practically unenforceable. +It would almost be easier to "regulate," "inspect," and "license" +the content of private mail--probably more so, since the mail system +is operated by the federal government. Boards are run by individuals, +independently, entirely at their own whim. + +For the sysop, the cost of operation is not the primary +limiting factor. Once the investment in a computer and +modem has been made, the only steady cost is the charge +for maintaining a phone line (or several phone lines). +The primary limits for sysops are time and energy. +Boards require upkeep. New users are generally "validated"-- +they must be issued individual passwords, and called at +home by voice-phone, so that their identity can be +verified. Obnoxious users, who exist in plenty, must be +chided or purged. Proliferating messages must be deleted +when they grow old, so that the capacity of the system +is not overwhelmed. And software programs (if such things +are kept on the board) must be examined for possible +computer viruses. If there is a financial charge to use +the board (increasingly common, especially in larger and +fancier systems) then accounts must be kept, and users +must be billed. And if the board crashes--a very common +occurrence--then repairs must be made. + +Boards can be distinguished by the amount of effort +spent in regulating them. First, we have the completely +open board, whose sysop is off chugging brews and +watching re-runs while his users generally degenerate +over time into peevish anarchy and eventual silence. +Second comes the supervised board, where the sysop +breaks in every once in a while to tidy up, calm brawls, +issue announcements, and rid the community of dolts +and troublemakers. Third is the heavily supervised +board, which sternly urges adult and responsible behavior +and swiftly edits any message considered offensive, +impertinent, illegal or irrelevant. And last comes +the completely edited "electronic publication," which +is presented to a silent audience which is not allowed +to respond directly in any way. + +Boards can also be grouped by their degree of anonymity. +There is the completely anonymous board, where everyone +uses pseudonyms--"handles"--and even the sysop is unaware +of the user's true identity. The sysop himself is likely +pseudonymous on a board of this type. Second, and rather +more common, is the board where the sysop knows (or thinks +he knows) the true names and addresses of all users, +but the users don't know one another's names and may not know his. +Third is the board where everyone has to use real names, +and roleplaying and pseudonymous posturing are forbidden. + +Boards can be grouped by their immediacy. "Chat-lines" +are boards linking several users together over several +different phone-lines simultaneously, so that people +exchange messages at the very moment that they type. +(Many large boards feature "chat" capabilities along +with other services.) Less immediate boards, +perhaps with a single phoneline, store messages serially, +one at a time. And some boards are only open for business +in daylight hours or on weekends, which greatly slows response. +A NETWORK of boards, such as "FidoNet," can carry electronic mail +from board to board, continent to continent, across huge distances-- +but at a relative snail's pace, so that a message can take several +days to reach its target audience and elicit a reply. + +Boards can be grouped by their degree of community. +Some boards emphasize the exchange of private, +person-to-person electronic mail. Others emphasize +public postings and may even purge people who "lurk," +merely reading posts but refusing to openly participate. +Some boards are intimate and neighborly. Others are frosty +and highly technical. Some are little more than storage +dumps for software, where users "download" and "upload" programs, +but interact among themselves little if at all. + +Boards can be grouped by their ease of access. Some boards +are entirely public. Others are private and restricted only +to personal friends of the sysop. Some boards divide users by status. +On these boards, some users, especially beginners, strangers or children, +will be restricted to general topics, and perhaps forbidden to post. +Favored users, though, are granted the ability to post as they please, +and to stay "on-line" as long as they like, even to the disadvantage +of other people trying to call in. High-status users can be given access +to hidden areas in the board, such as off-color topics, private discussions, +and/or valuable software. Favored users may even become "remote sysops" +with the power to take remote control of the board through their own +home computers. Quite often "remote sysops" end up doing all the work +and taking formal control of the enterprise, despite the fact that it's +physically located in someone else's house. Sometimes several "co-sysops" +share power. + +And boards can also be grouped by size. Massive, nationwide +commercial networks, such as CompuServe, Delphi, GEnie and Prodigy, +are run on mainframe computers and are generally not considered "boards," +though they share many of their characteristics, such as electronic mail, +discussion topics, libraries of software, and persistent and growing problems +with civil-liberties issues. Some private boards have as many as +thirty phone-lines and quite sophisticated hardware. And then +there are tiny boards. + +Boards vary in popularity. Some boards are huge and crowded, +where users must claw their way in against a constant busy-signal. +Others are huge and empty--there are few things sadder than a formerly +flourishing board where no one posts any longer, and the dead conversations +of vanished users lie about gathering digital dust. Some boards are tiny +and intimate, their telephone numbers intentionally kept confidential +so that only a small number can log on. + +And some boards are UNDERGROUND. + +Boards can be mysterious entities. The activities of +their users can be hard to differentiate from conspiracy. +Sometimes they ARE conspiracies. Boards have harbored, +or have been accused of harboring, all manner of fringe groups, +and have abetted, or been accused of abetting, every manner +of frowned-upon, sleazy, radical, and criminal activity. +There are Satanist boards. Nazi boards. Pornographic boards. +Pedophile boards. Drug- dealing boards. Anarchist boards. +Communist boards. Gay and Lesbian boards (these exist in great profusion, +many of them quite lively with well-established histories). +Religious cult boards. Evangelical boards. Witchcraft +boards, hippie boards, punk boards, skateboarder boards. +Boards for UFO believers. There may well be boards for +serial killers, airline terrorists and professional assassins. +There is simply no way to tell. Boards spring up, flourish, +and disappear in large numbers, in most every corner of +the developed world. Even apparently innocuous public +boards can, and sometimes do, harbor secret areas known +only to a few. And even on the vast, public, commercial services, +private mail is very private--and quite possibly criminal. + +Boards cover most every topic imaginable and some +that are hard to imagine. They cover a vast spectrum +of social activity. However, all board users do have +something in common: their possession of computers +and phones. Naturally, computers and phones are +primary topics of conversation on almost every board. + +And hackers and phone phreaks, those utter devotees +of computers and phones, live by boards. They swarm by boards. +They are bred by boards. By the late 1980s, phone-phreak groups +and hacker groups, united by boards, had proliferated fantastically. + + +As evidence, here is a list of hacker groups compiled +by the editors of Phrack on August 8, 1988. + + +The Administration. +Advanced Telecommunications, Inc. +ALIAS. +American Tone Travelers. +Anarchy Inc. +Apple Mafia. +The Association. +Atlantic Pirates Guild. + +Bad Ass Mother Fuckers. +Bellcore. +Bell Shock Force. +Black Bag. + +Camorra. +C&M Productions. +Catholics Anonymous. +Chaos Computer Club. +Chief Executive Officers. +Circle Of Death. +Circle Of Deneb. +Club X. +Coalition of Hi-Tech +Pirates. +Coast-To-Coast. +Corrupt Computing. +Cult Of The +Dead Cow. +Custom Retaliations. + +Damage Inc. +D&B Communications. +The Danger Gang. +Dec Hunters. +Digital Gang. +DPAK. + +Eastern Alliance. +The Elite Hackers Guild. +Elite Phreakers and Hackers Club. +The Elite Society Of America. +EPG. +Executives Of Crime. +Extasyy Elite. + +Fargo 4A. +Farmers Of Doom. +The Federation. +Feds R Us. +First Class. +Five O. +Five Star. +Force Hackers. +The 414s. + +Hack-A-Trip. +Hackers Of America. +High Mountain Hackers. +High Society. +The Hitchhikers. + +IBM Syndicate. +The Ice Pirates. +Imperial Warlords. +Inner Circle. +Inner Circle II. +Insanity Inc. +International Computer Underground Bandits. + +Justice League of America. + +Kaos Inc. +Knights Of Shadow. +Knights Of The Round Table. + +League Of Adepts. +Legion Of Doom. +Legion Of Hackers. +Lords Of Chaos. +Lunatic Labs, Unlimited. + +Master Hackers. +MAD! +The Marauders. +MD/PhD. + +Metal Communications, Inc. +MetalliBashers, Inc. +MBI. + +Metro Communications. +Midwest Pirates Guild. + +NASA Elite. +The NATO Association. +Neon Knights. + +Nihilist Order. +Order Of The Rose. +OSS. + +Pacific Pirates Guild. +Phantom Access Associates. + +PHido PHreaks. +The Phirm. +Phlash. +PhoneLine Phantoms. +Phone Phreakers Of America. +Phortune 500. + +Phreak Hack Delinquents. +Phreak Hack Destroyers. + +Phreakers, Hackers, And Laundromat Employees Gang (PHALSE Gang). +Phreaks Against Geeks. +Phreaks Against Phreaks Against Geeks. +Phreaks and Hackers of America. +Phreaks Anonymous World Wide. +Project Genesis. +The Punk Mafia. + +The Racketeers. +Red Dawn Text Files. +Roscoe Gang. + + +SABRE. +Secret Circle of Pirates. +Secret Service. +707 Club. +Shadow Brotherhood. +Sharp Inc. +65C02 Elite. + +Spectral Force. +Star League. +Stowaways. +Strata-Crackers. + + +Team Hackers '86. +Team Hackers '87. + +TeleComputist Newsletter Staff. +Tribunal Of Knowledge. + +Triple Entente. +Turn Over And Die Syndrome (TOADS). + +300 Club. +1200 Club. +2300 Club. +2600 Club. +2601 Club. + +2AF. + +The United Soft WareZ Force. +United Technical Underground. + +Ware Brigade. +The Warelords. +WASP. + +Contemplating this list is an impressive, almost humbling business. +As a cultural artifact, the thing approaches poetry. + +Underground groups--subcultures--can be distinguished +from independent cultures by their habit of referring +constantly to the parent society. Undergrounds by their +nature constantly must maintain a membrane of differentiation. +Funny/distinctive clothes and hair, specialized jargon, specialized +ghettoized areas in cities, different hours of rising, working, +sleeping. . . . The digital underground, which specializes in information, +relies very heavily on language to distinguish itself. As can be seen +from this list, they make heavy use of parody and mockery. +It's revealing to see who they choose to mock. + +First, large corporations. We have the Phortune 500, +The Chief Executive Officers, Bellcore, IBM Syndicate, +SABRE (a computerized reservation service maintained +by airlines). The common use of "Inc." is telling-- +none of these groups are actual corporations, +but take clear delight in mimicking them. + +Second, governments and police. NASA Elite, NATO Association. +"Feds R Us" and "Secret Service" are fine bits of fleering boldness. +OSS--the Office of Strategic Services was the forerunner of the CIA. + +Third, criminals. Using stigmatizing pejoratives as a perverse +badge of honor is a time-honored tactic for subcultures: +punks, gangs, delinquents, mafias, pirates, bandits, racketeers. + +Specialized orthography, especially the use of "ph" for "f" +and "z" for the plural "s," are instant recognition symbols. +So is the use of the numeral "0" for the letter "O" +--computer-software orthography generally features a +slash through the zero, making the distinction obvious. + +Some terms are poetically descriptive of computer intrusion: +the Stowaways, the Hitchhikers, the PhoneLine Phantoms, Coast-to-Coast. +Others are simple bravado and vainglorious puffery. +(Note the insistent use of the terms "elite" and "master.") +Some terms are blasphemous, some obscene, others merely cryptic-- +anything to puzzle, offend, confuse, and keep the straights at bay. + +Many hacker groups further re-encrypt their names +by the use of acronyms: United Technical Underground +becomes UTU, Farmers of Doom become FoD, the United SoftWareZ +Force becomes, at its own insistence, "TuSwF," and woe to the +ignorant rodent who capitalizes the wrong letters. + +It should be further recognized that the members of these groups +are themselves pseudonymous. If you did, in fact, run across +the "PhoneLine Phantoms," you would find them to consist of +"Carrier Culprit," "The Executioner," "Black Majik," +"Egyptian Lover," "Solid State," and "Mr Icom." +"Carrier Culprit" will likely be referred to by his friends +as "CC," as in, "I got these dialups from CC of PLP." + +It's quite possible that this entire list refers to as +few as a thousand people. It is not a complete list +of underground groups--there has never been such a list, +and there never will be. Groups rise, flourish, decline, +share membership, maintain a cloud of wannabes and +casual hangers-on. People pass in and out, are ostracized, +get bored, are busted by police, or are cornered by telco +security and presented with huge bills. Many "underground +groups" are software pirates, "warez d00dz," who might break +copy protection and pirate programs, but likely wouldn't dare +to intrude on a computer-system. + +It is hard to estimate the true population of the digital +underground. There is constant turnover. Most hackers +start young, come and go, then drop out at age 22-- +the age of college graduation. And a large majority +of "hackers" access pirate boards, adopt a handle, +swipe software and perhaps abuse a phone-code or two, +while never actually joining the elite. + +Some professional informants, who make it their business +to retail knowledge of the underground to paymasters in private +corporate security, have estimated the hacker population +at as high as fifty thousand. This is likely highly inflated, +unless one counts every single teenage software pirate +and petty phone-booth thief. My best guess is about 5,000 people. +Of these, I would guess that as few as a hundred are truly "elite" +--active computer intruders, skilled enough to penetrate +sophisticated systems and truly to worry corporate security +and law enforcement. + +Another interesting speculation is whether this group +is growing or not. Young teenage hackers are often +convinced that hackers exist in vast swarms and will soon +dominate the cybernetic universe. Older and wiser +veterans, perhaps as wizened as 24 or 25 years old, +are convinced that the glory days are long gone, that the cops +have the underground's number now, and that kids these days +are dirt-stupid and just want to play Nintendo. + +My own assessment is that computer intrusion, as a non-profit act +of intellectual exploration and mastery, is in slow decline, +at least in the United States; but that electronic fraud, +especially telecommunication crime, is growing by leaps and bounds. + +One might find a useful parallel to the digital underground +in the drug underground. There was a time, now much-obscured +by historical revisionism, when Bohemians freely shared joints +at concerts, and hip, small-scale marijuana dealers might +turn people on just for the sake of enjoying a long stoned conversation +about the Doors and Allen Ginsberg. Now drugs are increasingly verboten, +except in a high-stakes, highly-criminal world of highly addictive drugs. +Over years of disenchantment and police harassment, a vaguely ideological, +free-wheeling drug underground has relinquished the business of drug-dealing +to a far more savage criminal hard-core. This is not a pleasant prospect +to contemplate, but the analogy is fairly compelling. + +What does an underground board look like? What distinguishes +it from a standard board? It isn't necessarily the conversation-- +hackers often talk about common board topics, such as hardware, software, +sex, science fiction, current events, politics, movies, personal gossip. +Underground boards can best be distinguished by their files, or "philes," +pre-composed texts which teach the techniques and ethos of the underground. +These are prized reservoirs of forbidden knowledge. Some are anonymous, +but most proudly bear the handle of the "hacker" who has created them, +and his group affiliation, if he has one. + +Here is a partial table-of-contents of philes from an underground board, +somewhere in the heart of middle America, circa 1991. The descriptions +are mostly self-explanatory. + + +BANKAMER.ZIP 5406 06-11-91 Hacking Bank America +CHHACK.ZIP 4481 06-11-91 Chilton Hacking +CITIBANK.ZIP 4118 06-11-91 Hacking Citibank +CREDIMTC.ZIP 3241 06-11-91 Hacking Mtc Credit Company +DIGEST.ZIP 5159 06-11-91 Hackers Digest +HACK.ZIP 14031 06-11-91 How To Hack +HACKBAS.ZIP 5073 06-11-91 Basics Of Hacking +HACKDICT.ZIP 42774 06-11-91 Hackers Dictionary +HACKER.ZIP 57938 06-11-91 Hacker Info +HACKERME.ZIP 3148 06-11-91 Hackers Manual +HACKHAND.ZIP 4814 06-11-91 Hackers Handbook +HACKTHES.ZIP 48290 06-11-91 Hackers Thesis +HACKVMS.ZIP 4696 06-11-91 Hacking Vms Systems +MCDON.ZIP 3830 06-11-91 Hacking Macdonalds (Home Of The Archs) +P500UNIX.ZIP 15525 06-11-91 Phortune 500 Guide To Unix +RADHACK.ZIP 8411 06-11-91 Radio Hacking +TAOTRASH.DOC 4096 12-25-89 Suggestions For Trashing +TECHHACK.ZIP 5063 06-11-91 Technical Hacking + + +The files above are do-it-yourself manuals about computer intrusion. +The above is only a small section of a much larger library of hacking +and phreaking techniques and history. We now move into a different +and perhaps surprising area. + ++------------+ + |Anarchy| ++------------+ + +ANARC.ZIP 3641 06-11-91 Anarchy Files +ANARCHST.ZIP 63703 06-11-91 Anarchist Book +ANARCHY.ZIP 2076 06-11-91 Anarchy At Home +ANARCHY3.ZIP 6982 06-11-91 Anarchy No 3 +ANARCTOY.ZIP 2361 06-11-91 Anarchy Toys +ANTIMODM.ZIP 2877 06-11-91 Anti-modem Weapons +ATOM.ZIP 4494 06-11-91 How To Make An Atom Bomb +BARBITUA.ZIP 3982 06-11-91 Barbiturate Formula +BLCKPWDR.ZIP 2810 06-11-91 Black Powder Formulas +BOMB.ZIP 3765 06-11-91 How To Make Bombs +BOOM.ZIP 2036 06-11-91 Things That Go Boom +CHLORINE.ZIP 1926 06-11-91 Chlorine Bomb +COOKBOOK.ZIP 1500 06-11-91 Anarchy Cook Book +DESTROY.ZIP 3947 06-11-91 Destroy Stuff +DUSTBOMB.ZIP 2576 06-11-91 Dust Bomb +ELECTERR.ZIP 3230 06-11-91 Electronic Terror +EXPLOS1.ZIP 2598 06-11-91 Explosives 1 +EXPLOSIV.ZIP 18051 06-11-91 More Explosives +EZSTEAL.ZIP 4521 06-11-91 Ez-stealing +FLAME.ZIP 2240 06-11-91 Flame Thrower +FLASHLT.ZIP 2533 06-11-91 Flashlight Bomb +FMBUG.ZIP 2906 06-11-91 How To Make An Fm Bug +OMEEXPL.ZIP 2139 06-11-91 Home Explosives +HOW2BRK.ZIP 3332 06-11-91 How To Break In +LETTER.ZIP 2990 06-11-91 Letter Bomb +LOCK.ZIP 2199 06-11-91 How To Pick Locks +MRSHIN.ZIP 3991 06-11-91 Briefcase Locks +NAPALM.ZIP 3563 06-11-91 Napalm At Home +NITRO.ZIP 3158 06-11-91 Fun With Nitro +PARAMIL.ZIP 2962 06-11-91 Paramilitary Info +PICKING.ZIP 3398 06-11-91 Picking Locks +PIPEBOMB.ZIP 2137 06-11-91 Pipe Bomb +POTASS.ZIP 3987 06-11-91 Formulas With Potassium +PRANK.TXT 11074 08-03-90 More Pranks To Pull On Idiots! +REVENGE.ZIP 4447 06-11-91 Revenge Tactics +ROCKET.ZIP 2590 06-11-91 Rockets For Fun +SMUGGLE.ZIP 3385 06-11-91 How To Smuggle + +HOLY COW! The damned thing is full of stuff about bombs! + +What are we to make of this? + +First, it should be acknowledged that spreading +knowledge about demolitions to teenagers is a highly and +deliberately antisocial act. It is not, however, illegal. + +Second, it should be recognized that most of these +philes were in fact WRITTEN by teenagers. Most adult +American males who can remember their teenage years +will recognize that the notion of building a flamethrower +in your garage is an incredibly neat-o idea. ACTUALLY, +building a flamethrower in your garage, however, is +fraught with discouraging difficulty. Stuffing gunpowder +into a booby-trapped flashlight, so as to blow the arm off +your high-school vice-principal, can be a thing of dark +beauty to contemplate. Actually committing assault by +explosives will earn you the sustained attention of the +federal Bureau of Alcohol, Tobacco and Firearms. + +Some people, however, will actually try these plans. +A determinedly murderous American teenager can probably +buy or steal a handgun far more easily than he can brew +fake "napalm" in the kitchen sink. Nevertheless, +if temptation is spread before people, a certain number +will succumb, and a small minority will actually attempt +these stunts. A large minority of that small minority +will either fail or, quite likely, maim themselves, +since these "philes" have not been checked for accuracy, +are not the product of professional experience, +and are often highly fanciful. But the gloating menace +of these philes is not to be entirely dismissed. + +Hackers may not be "serious" about bombing; if they were, +we would hear far more about exploding flashlights, homemade bazookas, +and gym teachers poisoned by chlorine and potassium. +However, hackers are VERY serious about forbidden knowledge. +They are possessed not merely by curiosity, but by +a positive LUST TO KNOW. The desire to know what +others don't is scarcely new. But the INTENSITY +of this desire, as manifested by these young technophilic +denizens of the Information Age, may in fact BE new, +and may represent some basic shift in social values-- +a harbinger of what the world may come to, as society +lays more and more value on the possession, +assimilation and retailing of INFORMATION +as a basic commodity of daily life. + +There have always been young men with obsessive interests +in these topics. Never before, however, have they been able +to network so extensively and easily, and to propagandize +their interests with impunity to random passers-by. +High-school teachers will recognize that there's always +one in a crowd, but when the one in a crowd escapes control +by jumping into the phone-lines, and becomes a hundred such kids +all together on a board, then trouble is brewing visibly. +The urge of authority to DO SOMETHING, even something drastic, +is hard to resist. And in 1990, authority did something. +In fact authority did a great deal. + +# + +The process by which boards create hackers goes something +like this. A youngster becomes interested in computers-- +usually, computer games. He hears from friends that +"bulletin boards" exist where games can be obtained for free. +(Many computer games are "freeware," not copyrighted-- +invented simply for the love of it and given away to the public; +some of these games are quite good.) He bugs his parents for a modem, +or quite often, uses his parents' modem. + +The world of boards suddenly opens up. Computer games +can be quite expensive, real budget-breakers for a kid, +but pirated games, stripped of copy protection, are cheap or free. +They are also illegal, but it is very rare, almost unheard of, +for a small-scale software pirate to be prosecuted. +Once "cracked" of its copy protection, the program, +being digital data, becomes infinitely reproducible. +Even the instructions to the game, any manuals that accompany it, +can be reproduced as text files, or photocopied from legitimate sets. +Other users on boards can give many useful hints in game-playing tactics. +And a youngster with an infinite supply of free computer games can +certainly cut quite a swath among his modem-less friends. + +And boards are pseudonymous. No one need know that you're +fourteen years old--with a little practice at subterfuge, +you can talk to adults about adult things, and be accepted +and taken seriously! You can even pretend to be a girl, +or an old man, or anybody you can imagine. If you find this +kind of deception gratifying, there is ample opportunity +to hone your ability on boards. + +But local boards can grow stale. And almost every board maintains +a list of phone-numbers to other boards, some in distant, tempting, +exotic locales. Who knows what they're up to, in Oregon or Alaska +or Florida or California? It's very easy to find out--just order +the modem to call through its software--nothing to this, just typing +on a keyboard, the same thing you would do for most any computer game. +The machine reacts swiftly and in a few seconds you are talking to +a bunch of interesting people on another seaboard. + +And yet the BILLS for this trivial action can be staggering! +Just by going tippety-tap with your fingers, you may have +saddled your parents with four hundred bucks in long-distance charges, +and gotten chewed out but good. That hardly seems fair. + +How horrifying to have made friends in another state +and to be deprived of their company--and their software-- +just because telephone companies demand absurd amounts of money! +How painful, to be restricted to boards in one's own AREA CODE-- +what the heck is an "area code" anyway, and what makes it so special? +A few grumbles, complaints, and innocent questions of this sort +will often elicit a sympathetic reply from another board user-- +someone with some stolen codes to hand. You dither a while, +knowing this isn't quite right, then you make up your mind +to try them anyhow--AND THEY WORK! Suddenly you're doing something +even your parents can't do. Six months ago you were just some kid--now, +you're the Crimson Flash of Area Code 512! You're bad--you're nationwide! + +Maybe you'll stop at a few abused codes. Maybe you'll decide that +boards aren't all that interesting after all, that it's wrong, +not worth the risk --but maybe you won't. The next step +is to pick up your own repeat-dialling program-- +to learn to generate your own stolen codes. +(This was dead easy five years ago, much harder +to get away with nowadays, but not yet impossible.) +And these dialling programs are not complex or intimidating-- +some are as small as twenty lines of software. + +Now, you too can share codes. You can trade codes to learn +other techniques. If you're smart enough to catch on, +and obsessive enough to want to bother, and ruthless enough +to start seriously bending rules, then you'll get better, fast. +You start to develop a rep. You move up to a heavier class +of board--a board with a bad attitude, the kind of board +that naive dopes like your classmates and your former self +have never even heard of! You pick up the jargon of phreaking +and hacking from the board. You read a few of those anarchy philes-- +and man, you never realized you could be a real OUTLAW without +ever leaving your bedroom. + +You still play other computer games, but now you have a new +and bigger game. This one will bring you a different kind of status +than destroying even eight zillion lousy space invaders. + +Hacking is perceived by hackers as a "game." This is +not an entirely unreasonable or sociopathic perception. +You can win or lose at hacking, succeed or fail, +but it never feels "real." It's not simply that +imaginative youngsters sometimes have a hard time +telling "make-believe" from "real life." Cyberspace +is NOT REAL! "Real" things are physical objects +like trees and shoes and cars. Hacking takes place +on a screen. Words aren't physical, numbers +(even telephone numbers and credit card numbers) +aren't physical. Sticks and stones may break my bones, +but data will never hurt me. Computers SIMULATE reality, +like computer games that simulate tank battles or dogfights +or spaceships. Simulations are just make-believe, +and the stuff in computers is NOT REAL. + +Consider this: if "hacking" is supposed to be so serious and +real-life and dangerous, then how come NINE-YEAR-OLD KIDS have +computers and modems? You wouldn't give a nine year old his own car, +or his own rifle, or his own chainsaw--those things are "real." + +People underground are perfectly aware that the "game" +is frowned upon by the powers that be. Word gets around +about busts in the underground. Publicizing busts is one +of the primary functions of pirate boards, but they also +promulgate an attitude about them, and their own idiosyncratic +ideas of justice. The users of underground boards won't complain +if some guy is busted for crashing systems, spreading viruses, +or stealing money by wire-fraud. They may shake their heads +with a sneaky grin, but they won't openly defend these practices. +But when a kid is charged with some theoretical amount of theft: +$233,846.14, for instance, because he sneaked into a computer +and copied something, and kept it in his house on a floppy disk-- +this is regarded as a sign of near-insanity from prosecutors, +a sign that they've drastically mistaken the immaterial game +of computing for their real and boring everyday world +of fatcat corporate money. + +It's as if big companies and their suck-up lawyers +think that computing belongs to them, and they can +retail it with price stickers, as if it were boxes +of laundry soap! But pricing "information" is like +trying to price air or price dreams. Well, anybody +on a pirate board knows that computing can be, +and ought to be, FREE. Pirate boards are little +independent worlds in cyberspace, and they don't belong +to anybody but the underground. Underground boards +aren't "brought to you by Procter & Gamble." + +To log on to an underground board can mean to +experience liberation, to enter a world where, +for once, money isn't everything and adults +don't have all the answers. + +Let's sample another vivid hacker manifesto. Here are +some excerpts from "The Conscience of a Hacker," by "The Mentor," +from Phrack Volume One, Issue 7, Phile 3. + +"I made a discovery today. I found a computer. +Wait a second, this is cool. It does what I want it to. +If it makes a mistake, it's because I screwed it up. +Not because it doesn't like me. (. . .) +"And then it happened. . .a door opened to a world. . . +rushing through the phone line like heroin through an +addict's veins, an electronic pulse is sent out, +a refuge from day-to-day incompetencies is sought. . . +a board is found. `This is it. . .this is where I belong. . .' +"I know everyone here. . .even if I've never met them, +never talked to them, may never hear from them again. . . +I know you all. . . (. . .) + +"This is our world now. . .the world of the electron +and the switch, the beauty of the baud. We make use of a +service already existing without paying for what could be +dirt-cheap if it wasn't run by profiteering gluttons, and you +call us criminals. We explore. . .and you call us criminals. +We seek after knowledge. . .and you call us criminals. +We exist without skin color, without nationality, +without religious bias. . .and you call us criminals. +You build atomic bombs, you wage wars, you murder, +cheat and lie to us and try to make us believe that +it's for our own good, yet we're the criminals. + +"Yes, I am a criminal. My crime is that of curiosity. +My crime is that of judging people by what they say and think, +not what they look like. My crime is that of outsmarting you, +something that you will never forgive me for." + +# + +There have been underground boards almost as long +as there have been boards. One of the first was 8BBS, +which became a stronghold of the West Coast phone-phreak elite. +After going on-line in March 1980, 8BBS sponsored "Susan Thunder," +and "Tuc," and, most notoriously, "the Condor." "The Condor" +bore the singular distinction of becoming the most vilified +American phreak and hacker ever. Angry underground associates, +fed up with Condor's peevish behavior, turned him in to police, +along with a heaping double-helping of outrageous hacker legendry. +As a result, Condor was kept in solitary confinement for seven months, +for fear that he might start World War Three by triggering missile silos +from the prison payphone. (Having served his time, Condor is now +walking around loose; WWIII has thus far conspicuously failed to occur.) + +The sysop of 8BBS was an ardent free-speech enthusiast +who simply felt that ANY attempt to restrict the expression +of his users was unconstitutional and immoral. +Swarms of the technically curious entered 8BBS +and emerged as phreaks and hackers, until, in 1982, +a friendly 8BBS alumnus passed the sysop a new modem +which had been purchased by credit-card fraud. +Police took this opportunity to seize the entire board +and remove what they considered an attractive nuisance. + +Plovernet was a powerful East Coast pirate board +that operated in both New York and Florida. +Owned and operated by teenage hacker "Quasi Moto," +Plovernet attracted five hundred eager users in 1983. +"Emmanuel Goldstein" was one-time co-sysop of Plovernet, +along with "Lex Luthor," founder of the "Legion of Doom" group. +Plovernet bore the signal honor of being the original home +of the "Legion of Doom," about which the reader will be hearing +a great deal, soon. + +"Pirate-80," or "P-80," run by a sysop known as "Scan-Man," +got into the game very early in Charleston, and continued +steadily for years. P-80 flourished so flagrantly that +even its most hardened users became nervous, and some +slanderously speculated that "Scan Man" must have ties +to corporate security, a charge he vigorously denied. + +"414 Private" was the home board for the first GROUP +to attract conspicuous trouble, the teenage "414 Gang," +whose intrusions into Sloan-Kettering Cancer Center and +Los Alamos military computers were to be a nine-days-wonder in 1982. + +At about this time, the first software piracy boards +began to open up, trading cracked games for the Atari 800 +and the Commodore C64. Naturally these boards were +heavily frequented by teenagers. And with the 1983 +release of the hacker-thriller movie War Games, +the scene exploded. It seemed that every kid +in America had demanded and gotten a modem for Christmas. +Most of these dabbler wannabes put their modems in the attic +after a few weeks, and most of the remainder minded their +P's and Q's and stayed well out of hot water. But some +stubborn and talented diehards had this hacker kid in +War Games figured for a happening dude. They simply +could not rest until they had contacted the underground-- +or, failing that, created their own. + +In the mid-80s, underground boards sprang up like digital fungi. +ShadowSpawn Elite. Sherwood Forest I, II, and III. +Digital Logic Data Service in Florida, sysoped by no less +a man than "Digital Logic" himself; Lex Luthor of the +Legion of Doom was prominent on this board, since it +was in his area code. Lex's own board, "Legion of Doom," +started in 1984. The Neon Knights ran a network of Apple- +hacker boards: Neon Knights North, South, East and West. +Free World II was run by "Major Havoc." Lunatic Labs +is still in operation as of this writing. Dr. Ripco +in Chicago, an anything-goes anarchist board with an +extensive and raucous history, was seized by Secret Service +agents in 1990 on Sundevil day, but up again almost immediately, +with new machines and scarcely diminished vigor. + +The St. Louis scene was not to rank with major centers +of American hacking such as New York and L.A. But St. +Louis did rejoice in possession of "Knight Lightning" +and "Taran King," two of the foremost JOURNALISTS native +to the underground. Missouri boards like Metal Shop, +Metal Shop Private, Metal Shop Brewery, may not have +been the heaviest boards around in terms of illicit +expertise. But they became boards where hackers could +exchange social gossip and try to figure out what the +heck was going on nationally--and internationally. +Gossip from Metal Shop was put into the form of news files, +then assembled into a general electronic publication, +Phrack, a portmanteau title coined from "phreak" and "hack." +The Phrack editors were as obsessively curious about other +hackers as hackers were about machines. + +Phrack, being free of charge and lively reading, began +to circulate throughout the underground. As Taran King +and Knight Lightning left high school for college, +Phrack began to appear on mainframe machines linked to BITNET, +and, through BITNET to the "Internet," that loose but +extremely potent not-for-profit network where academic, +governmental and corporate machines trade data through +the UNIX TCP/IP protocol. (The "Internet Worm" of +November 2-3,1988, created by Cornell grad student Robert Morris, +was to be the largest and best-publicized computer-intrusion scandal +to date. Morris claimed that his ingenious "worm" program was meant +to harmlessly explore the Internet, but due to bad programming, +the Worm replicated out of control and crashed some six thousand +Internet computers. Smaller-scale and less ambitious Internet hacking +was a standard for the underground elite.) + +Most any underground board not hopelessly lame and out-of-it +would feature a complete run of Phrack--and, possibly, +the lesser-known standards of the underground: +the Legion of Doom Technical Journal, the obscene +and raucous Cult of the Dead Cow files, P/HUN magazine, +Pirate, the Syndicate Reports, and perhaps the highly +anarcho-political Activist Times Incorporated. + +Possession of Phrack on one's board was prima facie +evidence of a bad attitude. Phrack was seemingly everywhere, +aiding, abetting, and spreading the underground ethos. +And this did not escape the attention of corporate security +or the police. + +We now come to the touchy subject of police and boards. +Police, do, in fact, own boards. In 1989, there were +police-sponsored boards in California, Colorado, Florida, +Georgia, Idaho, Michigan, Missouri, Texas, and Virginia: +boards such as "Crime Bytes," "Crimestoppers," "All Points" +and "Bullet-N-Board." Police officers, as private computer +enthusiasts, ran their own boards in Arizona, California, +Colorado, Connecticut, Florida, Missouri, Maryland, +New Mexico, North Carolina, Ohio, Tennessee and Texas. +Police boards have often proved helpful in community relations. +Sometimes crimes are reported on police boards. + +Sometimes crimes are COMMITTED on police boards. +This has sometimes happened by accident, as naive hackers +blunder onto police boards and blithely begin offering telephone codes. +Far more often, however, it occurs through the now almost-traditional +use of "sting boards." The first police sting-boards were established +in 1985: "Underground Tunnel" in Austin, Texas, whose sysop +Sgt. Robert Ansley called himself "Pluto"--"The Phone Company" +in Phoenix, Arizona, run by Ken MacLeod of the Maricopa County +Sheriff's office--and Sgt. Dan Pasquale's board in Fremont, California. +Sysops posed as hackers, and swiftly garnered coteries of ardent users, +who posted codes and loaded pirate software with abandon, +and came to a sticky end. + +Sting boards, like other boards, are cheap to operate, +very cheap by the standards of undercover police operations. +Once accepted by the local underground, sysops will likely be +invited into other pirate boards, where they can compile more dossiers. +And when the sting is announced and the worst offenders arrested, +the publicity is generally gratifying. The resultant paranoia +in the underground--perhaps more justly described as a "deterrence effect"-- +tends to quell local lawbreaking for quite a while. + +Obviously police do not have to beat the underbrush for hackers. +On the contrary, they can go trolling for them. Those caught +can be grilled. Some become useful informants. They can lead +the way to pirate boards all across the country. + +And boards all across the country showed the sticky +fingerprints of Phrack, and of that loudest and most +flagrant of all underground groups, the "Legion of Doom." + +The term "Legion of Doom" came from comic books. The Legion of Doom, +a conspiracy of costumed super- villains headed by the chrome-domed +criminal ultra- mastermind Lex Luthor, gave Superman a lot of four-color +graphic trouble for a number of decades. Of course, Superman, +that exemplar of Truth, Justice, and the American Way, +always won in the long run. This didn't matter to the hacker Doomsters-- +"Legion of Doom" was not some thunderous and evil Satanic reference, +it was not meant to be taken seriously. "Legion of Doom" came +from funny-books and was supposed to be funny. + +"Legion of Doom" did have a good mouthfilling ring to it, though. +It sounded really cool. Other groups, such as the "Farmers of Doom," +closely allied to LoD, recognized this grandiloquent quality, +and made fun of it. There was even a hacker group called +"Justice League of America," named after Superman's club +of true-blue crimefighting superheros. + +But they didn't last; the Legion did. + +The original Legion of Doom, hanging out on Quasi Moto's Plovernet board, +were phone phreaks. They weren't much into computers. "Lex Luthor" himself +(who was under eighteen when he formed the Legion) was a COSMOS expert, +COSMOS being the "Central System for Mainframe Operations," +a telco internal computer network. Lex would eventually become +quite a dab hand at breaking into IBM mainframes, but although +everyone liked Lex and admired his attitude, he was not considered +a truly accomplished computer intruder. Nor was he the "mastermind" +of the Legion of Doom--LoD were never big on formal leadership. +As a regular on Plovernet and sysop of his "Legion of Doom BBS," +Lex was the Legion's cheerleader and recruiting officer. + +Legion of Doom began on the ruins of an earlier phreak group, +The Knights of Shadow. Later, LoD was to subsume the personnel +of the hacker group "Tribunal of Knowledge." People came and went +constantly in LoD; groups split up or formed offshoots. + +Early on, the LoD phreaks befriended a few computer-intrusion +enthusiasts, who became the associated "Legion of Hackers." +Then the two groups conflated into the "Legion of Doom/Hackers," +or LoD/H. When the original "hacker" wing, Messrs. "Compu-Phreak" +and "Phucked Agent 04," found other matters to occupy their time, +the extra "/H" slowly atrophied out of the name; but by this time +the phreak wing, Messrs. Lex Luthor, "Blue Archer," "Gary Seven," +"Kerrang Khan," "Master of Impact," "Silver Spy," "The Marauder," +and "The Videosmith," had picked up a plethora of intrusion +expertise and had become a force to be reckoned with. + +LoD members seemed to have an instinctive understanding +that the way to real power in the underground lay through +covert publicity. LoD were flagrant. Not only was it one +of the earliest groups, but the members took pains to widely +distribute their illicit knowledge. Some LoD members, +like "The Mentor," were close to evangelical about it. +Legion of Doom Technical Journal began to show up on boards +throughout the underground. + +LoD Technical Journal was named in cruel parody +of the ancient and honored AT&T Technical Journal. +The material in these two publications was quite similar-- +much of it, adopted from public journals and discussions +in the telco community. And yet, the predatory attitude +of LoD made even its most innocuous data seem deeply sinister; +an outrage; a clear and present danger. + +To see why this should be, let's consider the following +(invented) paragraphs, as a kind of thought experiment. + +(A) "W. Fred Brown, AT&T Vice President for +Advanced Technical Development, testified May 8 +at a Washington hearing of the National Telecommunications +and Information Administration (NTIA), regarding +Bellcore's GARDEN project. GARDEN (Generalized +Automatic Remote Distributed Electronic Network) is a +telephone-switch programming tool that makes it possible +to develop new telecom services, including hold-on-hold +and customized message transfers, from any keypad terminal, +within seconds. The GARDEN prototype combines centrex +lines with a minicomputer using UNIX operating system software." + +(B) "Crimson Flash 512 of the Centrex Mobsters reports: +D00dz, you wouldn't believe this GARDEN bullshit Bellcore's +just come up with! Now you don't even need a lousy Commodore +to reprogram a switch--just log on to GARDEN as a technician, +and you can reprogram switches right off the keypad in any +public phone booth! You can give yourself hold-on-hold +and customized message transfers, and best of all, +the thing is run off (notoriously insecure) centrex lines +using--get this--standard UNIX software! Ha ha ha ha!" + +Message (A), couched in typical techno-bureaucratese, +appears tedious and almost unreadable. (A) scarcely seems +threatening or menacing. Message (B), on the other hand, +is a dreadful thing, prima facie evidence of a dire conspiracy, +definitely not the kind of thing you want your teenager reading. + +The INFORMATION, however, is identical. It is PUBLIC +information, presented before the federal government in +an open hearing. It is not "secret." It is not "proprietary." +It is not even "confidential." On the contrary, the +development of advanced software systems is a matter +of great public pride to Bellcore. + +However, when Bellcore publicly announces a project of this kind, +it expects a certain attitude from the public--something along +the lines of GOSH WOW, YOU GUYS ARE GREAT, KEEP THAT UP, WHATEVER IT IS-- +certainly not cruel mimickry, one-upmanship and outrageous speculations +about possible security holes. + +Now put yourself in the place of a policeman confronted by +an outraged parent, or telco official, with a copy of Version (B). +This well-meaning citizen, to his horror, has discovered +a local bulletin-board carrying outrageous stuff like (B), +which his son is examining with a deep and unhealthy interest. +If (B) were printed in a book or magazine, you, as an American +law enforcement officer, would know that it would take +a hell of a lot of trouble to do anything about it; +but it doesn't take technical genius to recognize that +if there's a computer in your area harboring stuff like (B), +there's going to be trouble. + +In fact, if you ask around, any computer-literate cop +will tell you straight out that boards with stuff like (B) +are the SOURCE of trouble. And the WORST source of trouble +on boards are the ringleaders inventing and spreading stuff like (B). +If it weren't for these jokers, there wouldn't BE any trouble. + +And Legion of Doom were on boards like nobody else. +Plovernet. The Legion of Doom Board. The Farmers of Doom Board. +Metal Shop. OSUNY. Blottoland. Private Sector. Atlantis. +Digital Logic. Hell Phrozen Over. + +LoD members also ran their own boards. "Silver Spy" started +his own board, "Catch-22," considered one of the heaviest around. +So did "Mentor," with his "Phoenix Project." When they didn't run boards +themselves, they showed up on other people's boards, to brag, boast, +and strut. And where they themselves didn't go, their philes went, +carrying evil knowledge and an even more evil attitude. + +As early as 1986, the police were under the vague impression +that EVERYONE in the underground was Legion of Doom. +LoD was never that large--considerably smaller than either +"Metal Communications" or "The Administration," for instance-- +but LoD got tremendous press. Especially in Phrack, +which at times read like an LoD fan magazine; and Phrack +was everywhere, especially in the offices of telco security. +You couldn't GET busted as a phone phreak, a hacker, +or even a lousy codes kid or warez dood, without the cops +asking if you were LoD. + +This was a difficult charge to deny, as LoD never +distributed membership badges or laminated ID cards. +If they had, they would likely have died out quickly, +for turnover in their membership was considerable. +LoD was less a high-tech street-gang than an ongoing +state-of-mind. LoD was the Gang That Refused to Die. +By 1990, LoD had RULED for ten years, and it seemed WEIRD +to police that they were continually busting people who were +only sixteen years old. All these teenage small-timers +were pleading the tiresome hacker litany of "just curious, +no criminal intent." Somewhere at the center of this +conspiracy there had to be some serious adult masterminds, +not this seemingly endless supply of myopic suburban +white kids with high SATs and funny haircuts. + +There was no question that most any American hacker +arrested would "know" LoD. They knew the handles +of contributors to LoD Tech Journal, and were likely +to have learned their craft through LoD boards and LoD activism. +But they'd never met anyone from LoD. Even some of the +rotating cadre who were actually and formally "in LoD" +knew one another only by board-mail and pseudonyms. +This was a highly unconventional profile for a criminal conspiracy. +Computer networking, and the rapid evolution of the digital underground, +made the situation very diffuse and confusing. + +Furthermore, a big reputation in the digital underground +did not coincide with one's willingness to commit "crimes." +Instead, reputation was based on cleverness and technical mastery. +As a result, it often seemed that the HEAVIER the hackers were, +the LESS likely they were to have committed any kind of common, +easily prosecutable crime. There were some hackers who could really steal. +And there were hackers who could really hack. But the two groups didn't seem +to overlap much, if at all. For instance, most people in the underground +looked up to "Emmanuel Goldstein" of 2600 as a hacker demigod. +But Goldstein's publishing activities were entirely legal-- +Goldstein just printed dodgy stuff and talked about politics, +he didn't even hack. When you came right down to it, +Goldstein spent half his time complaining that computer security +WASN'T STRONG ENOUGH and ought to be drastically improved +across the board! + +Truly heavy-duty hackers, those with serious technical skills +who had earned the respect of the underground, never stole money +or abused credit cards. Sometimes they might abuse phone-codes-- +but often, they seemed to get all the free phone-time they wanted +without leaving a trace of any kind. + +The best hackers, the most powerful and technically accomplished, +were not professional fraudsters. They raided computers habitually, +but wouldn't alter anything, or damage anything. They didn't even steal +computer equipment--most had day-jobs messing with hardware, +and could get all the cheap secondhand equipment they wanted. +The hottest hackers, unlike the teenage wannabes, weren't snobs +about fancy or expensive hardware. Their machines tended to be +raw second-hand digital hot-rods full of custom add-ons that +they'd cobbled together out of chickenwire, memory chips and spit. +Some were adults, computer software writers and consultants by trade, +and making quite good livings at it. Some of them ACTUALLY WORKED +FOR THE PHONE COMPANY--and for those, the "hackers" actually found +under the skirts of Ma Bell, there would be little mercy in 1990. + +It has long been an article of faith in the +underground that the "best" hackers never get caught. +They're far too smart, supposedly. They never get caught +because they never boast, brag, or strut. These demigods +may read underground boards (with a condescending smile), +but they never say anything there. The "best" hackers, +according to legend, are adult computer professionals, +such as mainframe system administrators, who already know +the ins and outs of their particular brand of security. +Even the "best" hacker can't break in to just any computer at random: +the knowledge of security holes is too specialized, varying widely +with different software and hardware. But if people are employed to run, +say, a UNIX mainframe or a VAX/VMS machine, then they tend to learn +security from the inside out. Armed with this knowledge, +they can look into most anybody else's UNIX or VMS +without much trouble or risk, if they want to. +And, according to hacker legend, of course they want to, +so of course they do. They just don't make a big deal +of what they've done. So nobody ever finds out. + +It is also an article of faith in the underground that +professional telco people "phreak" like crazed weasels. +OF COURSE they spy on Madonna's phone calls--I mean, +WOULDN'T YOU? Of course they give themselves free long- +distance--why the hell should THEY pay, they're running +the whole shebang! + +It has, as a third matter, long been an article of faith +that any hacker caught can escape serious punishment if +he confesses HOW HE DID IT. Hackers seem to believe +that governmental agencies and large corporations are +blundering about in cyberspace like eyeless jellyfish +or cave salamanders. They feel that these large +but pathetically stupid organizations will proffer up +genuine gratitude, and perhaps even a security post +and a big salary, to the hot-shot intruder who will deign +to reveal to them the supreme genius of his modus operandi. + +In the case of longtime LoD member "Control-C," +this actually happened, more or less. Control-C had led +Michigan Bell a merry chase, and when captured in 1987, +he turned out to be a bright and apparently physically +harmless young fanatic, fascinated by phones. There was +no chance in hell that Control-C would actually repay the +enormous and largely theoretical sums in long-distance +service that he had accumulated from Michigan Bell. +He could always be indicted for fraud or computer-intrusion, +but there seemed little real point in this--he hadn't +physically damaged any computer. He'd just plead guilty, +and he'd likely get the usual slap-on-the-wrist, +and in the meantime it would be a big hassle for Michigan Bell +just to bring up the case. But if kept on the payroll, +he might at least keep his fellow hackers at bay. + +There were uses for him. For instance, a contrite +Control-C was featured on Michigan Bell internal posters, +sternly warning employees to shred their trash. +He'd always gotten most of his best inside info from +"trashing"--raiding telco dumpsters, for useful data +indiscreetly thrown away. He signed these posters, too. +Control-C had become something like a Michigan Bell mascot. +And in fact, Control-C DID keep other hackers at bay. +Little hackers were quite scared of Control-C and his +heavy-duty Legion of Doom friends. And big hackers WERE +his friends and didn't want to screw up his cushy situation. + +No matter what one might say of LoD, they did stick together. +When "Wasp," an apparently genuinely malicious New York hacker, +began crashing Bellcore machines, Control-C received swift volunteer +help from "the Mentor" and the Georgia LoD wing made up of +"The Prophet," "Urvile," and "Leftist." Using Mentor's Phoenix +Project board to coordinate, the Doomsters helped telco security +to trap Wasp, by luring him into a machine with a tap +and line-trace installed. Wasp lost. LoD won! And my, did they brag. + +Urvile, Prophet and Leftist were well-qualified for this activity, +probably more so even than the quite accomplished Control-C. +The Georgia boys knew all about phone switching-stations. +Though relative johnny-come-latelies in the Legion of Doom, +they were considered some of LoD's heaviest guys, +into the hairiest systems around. They had the good fortune +to live in or near Atlanta, home of the sleepy and apparently +tolerant BellSouth RBOC. + +As RBOC security went, BellSouth were "cake." US West (of Arizona, +the Rockies and the Pacific Northwest) were tough and aggressive, +probably the heaviest RBOC around. Pacific Bell, California's PacBell, +were sleek, high-tech, and longtime veterans of the LA phone-phreak wars. +NYNEX had the misfortune to run the New York City area, and were warily +prepared for most anything. Even Michigan Bell, a division of the +Ameritech RBOC, at least had the elementary sense to hire their own hacker +as a useful scarecrow. But BellSouth, even though their corporate P.R. +proclaimed them to have "Everything You Expect From a Leader," were pathetic. + +When rumor about LoD's mastery of Georgia's switching network got around +to BellSouth through Bellcore and telco security scuttlebutt, +they at first refused to believe it. If you paid serious attention +to every rumor out and about these hacker kids, you would hear all kinds +of wacko saucer-nut nonsense: that the National Security Agency +monitored all American phone calls, that the CIA and DEA tracked +traffic on bulletin-boards with word-analysis programs, +that the Condor could start World War III from a payphone. + +If there were hackers into BellSouth switching-stations, then how come +nothing had happened? Nothing had been hurt. BellSouth's machines +weren't crashing. BellSouth wasn't suffering especially badly from fraud. +BellSouth's customers weren't complaining. BellSouth was headquartered +in Atlanta, ambitious metropolis of the new high-tech Sunbelt; +and BellSouth was upgrading its network by leaps and bounds, +digitizing the works left right and center. They could hardly be +considered sluggish or naive. BellSouth's technical expertise +was second to none, thank you kindly. But then came the Florida business. + +On June 13, 1989, callers to the Palm Beach County Probation Department, +in Delray Beach, Florida, found themselves involved in a remarkable +discussion with a phone-sex worker named "Tina" in New York State. +Somehow, ANY call to this probation office near Miami was instantly +and magically transported across state lines, at no extra charge to the user, +to a pornographic phone-sex hotline hundreds of miles away! + +This practical joke may seem utterly hilarious at first hearing, +and indeed there was a good deal of chuckling about it in +phone phreak circles, including the Autumn 1989 issue of 2600. +But for Southern Bell (the division of the BellSouth RBOC +supplying local service for Florida, Georgia, North Carolina +and South Carolina), this was a smoking gun. For the first time ever, +a computer intruder had broken into a BellSouth central office +switching station and re-programmed it! + +Or so BellSouth thought in June 1989. Actually, LoD members had been +frolicking harmlessly in BellSouth switches since September 1987. +The stunt of June 13--call-forwarding a number through manipulation +of a switching station--was child's play for hackers as accomplished +as the Georgia wing of LoD. Switching calls interstate sounded like +a big deal, but it took only four lines of code to accomplish this. +An easy, yet more discreet, stunt, would be to call-forward another +number to your own house. If you were careful and considerate, +and changed the software back later, then not a soul would know. +Except you. And whoever you had bragged to about it. + +As for BellSouth, what they didn't know wouldn't hurt them. + +Except now somebody had blown the whole thing wide open, and BellSouth knew. + +A now alerted and considerably paranoid BellSouth began searching switches +right and left for signs of impropriety, in that hot summer of 1989. +No fewer than forty-two BellSouth employees were put on 12-hour shifts, +twenty-four hours a day, for two solid months, poring over records +and monitoring computers for any sign of phony access. These forty-two +overworked experts were known as BellSouth's "Intrusion Task Force." + +What the investigators found astounded them. Proprietary telco databases +had been manipulated: phone numbers had been created out of thin air, +with no users' names and no addresses. And perhaps worst of all, +no charges and no records of use. The new digital ReMOB (Remote Observation) +diagnostic feature had been extensively tampered with--hackers had learned to +reprogram ReMOB software, so that they could listen in on any switch-routed +call at their leisure! They were using telco property to SPY! + +The electrifying news went out throughout law enforcement in 1989. +It had never really occurred to anyone at BellSouth that their prized +and brand-new digital switching-stations could be RE-PROGRAMMED. +People seemed utterly amazed that anyone could have the nerve. +Of course these switching stations were "computers," and everybody +knew hackers liked to "break into computers:" but telephone people's +computers were DIFFERENT from normal people's computers. + +The exact reason WHY these computers were "different" was +rather ill-defined. It certainly wasn't the extent of their security. +The security on these BellSouth computers was lousy; the AIMSX computers, +for instance, didn't even have passwords. But there was no question that +BellSouth strongly FELT that their computers were very different indeed. +And if there were some criminals out there who had not gotten that message, +BellSouth was determined to see that message taught. + +After all, a 5ESS switching station was no mere bookkeeping system for +some local chain of florists. Public service depended on these stations. +Public SAFETY depended on these stations. + +And hackers, lurking in there call-forwarding or ReMobbing, could spy +on anybody in the local area! They could spy on telco officials! +They could spy on police stations! They could spy on local offices +of the Secret Service. . . . + +In 1989, electronic cops and hacker-trackers began using scrambler-phones +and secured lines. It only made sense. There was no telling who was into +those systems. Whoever they were, they sounded scary. This was some +new level of antisocial daring. Could be West German hackers, in the pay +of the KGB. That too had seemed a weird and farfetched notion, +until Clifford Stoll had poked and prodded a sluggish Washington +law-enforcement bureaucracy into investigating a computer intrusion +that turned out to be exactly that--HACKERS, IN THE PAY OF THE KGB! +Stoll, the systems manager for an Internet lab in Berkeley California, +had ended up on the front page of the New Nork Times, proclaimed a national +hero in the first true story of international computer espionage. +Stoll's counterspy efforts, which he related in a bestselling book, +The Cuckoo's Egg, in 1989, had established the credibility of `hacking' +as a possible threat to national security. The United States Secret Service +doesn't mess around when it suspects a possible action by a foreign +intelligence apparat. + +The Secret Service scrambler-phones and secured lines put +a tremendous kink in law enforcement's ability to operate freely; +to get the word out, cooperate, prevent misunderstandings. +Nevertheless, 1989 scarcely seemed the time for half-measures. +If the police and Secret Service themselves were not operationally secure, +then how could they reasonably demand measures of security from +private enterprise? At least, the inconvenience made people aware +of the seriousness of the threat. + +If there was a final spur needed to get the police off the dime, +it came in the realization that the emergency 911 system was vulnerable. +The 911 system has its own specialized software, but it is run on the same +digital switching systems as the rest of the telephone network. +911 is not physically different from normal telephony. But it is +certainly culturally different, because this is the area of +telephonic cyberspace reserved for the police and emergency services. + +Your average policeman may not know much about hackers or phone-phreaks. +Computer people are weird; even computer COPS are rather weird; +the stuff they do is hard to figure out. But a threat to the 911 system +is anything but an abstract threat. If the 911 system goes, people can die. + +Imagine being in a car-wreck, staggering to a phone-booth, +punching 911 and hearing "Tina" pick up the phone-sex line +somewhere in New York! The situation's no longer comical, somehow. + +And was it possible? No question. Hackers had attacked 911 +systems before. Phreaks can max-out 911 systems just by siccing +a bunch of computer-modems on them in tandem, dialling them over +and over until they clog. That's very crude and low-tech, +but it's still a serious business. + +The time had come for action. It was time to take stern measures +with the underground. It was time to start picking up the dropped threads, +the loose edges, the bits of braggadocio here and there; it was time to get +on the stick and start putting serious casework together. Hackers weren't +"invisible." They THOUGHT they were invisible; but the truth was, +they had just been tolerated too long. + +Under sustained police attention in the summer of '89, the digital +underground began to unravel as never before. + +The first big break in the case came very early on: July 1989, +the following month. The perpetrator of the "Tina" switch was caught, +and confessed. His name was "Fry Guy," a 16-year-old in Indiana. +Fry Guy had been a very wicked young man. + +Fry Guy had earned his handle from a stunt involving French fries. +Fry Guy had filched the log-in of a local MacDonald's manager +and had logged-on to the MacDonald's mainframe on the Sprint +Telenet system. Posing as the manager, Fry Guy had altered +MacDonald's records, and given some teenage hamburger-flipping +friends of his, generous raises. He had not been caught. + +Emboldened by success, Fry Guy moved on to credit-card abuse. +Fry Guy was quite an accomplished talker; with a gift for +"social engineering." If you can do "social engineering" +--fast-talk, fake-outs, impersonation, conning, scamming-- +then card abuse comes easy. (Getting away with it in +the long run is another question). + +Fry Guy had run across "Urvile" of the Legion of Doom +on the ALTOS Chat board in Bonn, Germany. ALTOS Chat +was a sophisticated board, accessible through globe-spanning +computer networks like BITnet, Tymnet, and Telenet. +ALTOS was much frequented by members of Germany's +Chaos Computer Club. Two Chaos hackers who hung out on ALTOS, +"Jaeger" and "Pengo," had been the central villains of +Clifford Stoll's Cuckoo's Egg case: consorting in East Berlin +with a spymaster from the KGB, and breaking into American +computers for hire, through the Internet. + +When LoD members learned the story of Jaeger's depredations +from Stoll's book, they were rather less than impressed, +technically speaking. On LoD's own favorite board of the moment, +"Black Ice," LoD members bragged that they themselves could have done +all the Chaos break-ins in a week flat! Nevertheless, LoD were grudgingly +impressed by the Chaos rep, the sheer hairy-eyed daring of hash-smoking +anarchist hackers who had rubbed shoulders with the fearsome big-boys +of international Communist espionage. LoD members sometimes traded +bits of knowledge with friendly German hackers on ALTOS--phone numbers +for vulnerable VAX/VMS computers in Georgia, for instance. +Dutch and British phone phreaks, and the Australian clique of +"Phoenix," "Nom," and "Electron," were ALTOS regulars, too. +In underground circles, to hang out on ALTOS was considered +the sign of an elite dude, a sophisticated hacker of the +international digital jet-set. + +Fry Guy quickly learned how to raid information from credit-card +consumer-reporting agencies. He had over a hundred stolen credit-card +numbers in his notebooks, and upwards of a thousand swiped long-distance +access codes. He knew how to get onto Altos, and how to talk the talk of +the underground convincingly. He now wheedled knowledge of switching-station +tricks from Urvile on the ALTOS system. + +Combining these two forms of knowledge enabled Fry Guy to bootstrap +his way up to a new form of wire-fraud. First, he'd snitched credit card +numbers from credit-company computers. The data he copied included names, +addresses and phone numbers of the random card-holders. + +Then Fry Guy, impersonating a card-holder, called up Western Union +and asked for a cash advance on "his" credit card. Western Union, +as a security guarantee, would call the customer back, at home, +to verify the transaction. + +But, just as he had switched the Florida probation office to "Tina" +in New York, Fry Guy switched the card-holder's number to a local pay-phone. +There he would lurk in wait, muddying his trail by routing and re-routing +the call, through switches as far away as Canada. When the call came through, +he would boldly "social-engineer," or con, the Western Union people, pretending +to be the legitimate card-holder. Since he'd answered the proper phone number, +the deception was not very hard. Western Union's money was then shipped to +a confederate of Fry Guy's in his home town in Indiana. + +Fry Guy and his cohort, using LoD techniques, stole six thousand dollars +from Western Union between December 1988 and July 1989. They also dabbled +in ordering delivery of stolen goods through card-fraud. Fry Guy +was intoxicated with success. The sixteen-year-old fantasized wildly +to hacker rivals, boasting that he'd used rip-off money to hire himself +a big limousine, and had driven out-of-state with a groupie from +his favorite heavy-metal band, Motley Crue. + +Armed with knowledge, power, and a gratifying stream of free money, +Fry Guy now took it upon himself to call local representatives +of Indiana Bell security, to brag, boast, strut, and utter +tormenting warnings that his powerful friends in the notorious +Legion of Doom could crash the national telephone network. +Fry Guy even named a date for the scheme: the Fourth of July, +a national holiday. + +This egregious example of the begging-for-arrest syndrome was shortly +followed by Fry Guy's arrest. After the Indiana telephone company figured +out who he was, the Secret Service had DNRs--Dialed Number Recorders-- +installed on his home phone lines. These devices are not taps, and can't +record the substance of phone calls, but they do record the phone numbers +of all calls going in and out. Tracing these numbers showed Fry Guy's +long-distance code fraud, his extensive ties to pirate bulletin boards, +and numerous personal calls to his LoD friends in Atlanta. By July 11, +1989, Prophet, Urvile and Leftist also had Secret Service DNR +"pen registers" installed on their own lines. + +The Secret Service showed up in force at Fry Guy's house on July 22, 1989, +to the horror of his unsuspecting parents. The raiders were led by +a special agent from the Secret Service's Indianapolis office. +However, the raiders were accompanied and advised by Timothy M. Foley +of the Secret Service's Chicago office (a gentleman about whom +we will soon be hearing a great deal). + +Following federal computer-crime techniques that had been standard +since the early 1980s, the Secret Service searched the house thoroughly, +and seized all of Fry Guy's electronic equipment and notebooks. +All Fry Guy's equipment went out the door in the custody of the +Secret Service, which put a swift end to his depredations. + +The USSS interrogated Fry Guy at length. His case was put in the charge +of Deborah Daniels, the federal US Attorney for the Southern District +of Indiana. Fry Guy was charged with eleven counts of computer fraud, +unauthorized computer access, and wire fraud. The evidence was thorough +and irrefutable. For his part, Fry Guy blamed his corruption on the +Legion of Doom and offered to testify against them. + +Fry Guy insisted that the Legion intended to crash the phone system +on a national holiday. And when AT&T crashed on Martin Luther King Day, +1990, this lent a credence to his claim that genuinely alarmed telco +security and the Secret Service. + +Fry Guy eventually pled guilty on May 31, 1990. On September 14, +he was sentenced to forty-four months' probation and four hundred hours' +community service. He could have had it much worse; but it made sense +to prosecutors to take it easy on this teenage minor, while zeroing +in on the notorious kingpins of the Legion of Doom. + +But the case against LoD had nagging flaws. Despite the best effort +of investigators, it was impossible to prove that the Legion had crashed +the phone system on January 15, because they, in fact, hadn't done so. +The investigations of 1989 did show that certain members of +the Legion of Doom had achieved unprecedented power over the telco +switching stations, and that they were in active conspiracy +to obtain more power yet. Investigators were privately convinced +that the Legion of Doom intended to do awful things with this knowledge, +but mere evil intent was not enough to put them in jail. + +And although the Atlanta Three--Prophet, Leftist, and especially Urvile-- +had taught Fry Guy plenty, they were not themselves credit-card fraudsters. +The only thing they'd "stolen" was long-distance service--and since they'd +done much of that through phone-switch manipulation, there was no easy way +to judge how much they'd "stolen," or whether this practice was even "theft" +of any easily recognizable kind. + +Fry Guy's theft of long-distance codes had cost the phone companies plenty. +The theft of long-distance service may be a fairly theoretical "loss," +but it costs genuine money and genuine time to delete all those stolen codes, +and to re-issue new codes to the innocent owners of those corrupted codes. +The owners of the codes themselves are victimized, and lose time and money +and peace of mind in the hassle. And then there were the credit-card victims +to deal with, too, and Western Union. When it came to rip-off, Fry Guy was +far more of a thief than LoD. It was only when it came to actual computer +expertise that Fry Guy was small potatoes. + +The Atlanta Legion thought most "rules" of cyberspace were for rodents +and losers, but they DID have rules. THEY NEVER CRASHED ANYTHING, +AND THEY NEVER TOOK MONEY. These were rough rules-of-thumb, and +rather dubious principles when it comes to the ethical subtleties +of cyberspace, but they enabled the Atlanta Three to operate with +a relatively clear conscience (though never with peace of mind). + +If you didn't hack for money, if you weren't robbing people of actual funds +--money in the bank, that is-- then nobody REALLY got hurt, in LoD's opinion. +"Theft of service" was a bogus issue, and "intellectual property" was +a bad joke. But LoD had only elitist contempt for rip-off artists, +"leechers," thieves. They considered themselves clean. In their opinion, +if you didn't smash-up or crash any systems --(well, not on purpose, anyhow-- +accidents can happen, just ask Robert Morris) then it was very unfair +to call you a "vandal" or a "cracker." When you were hanging out on-line +with your "pals" in telco security, you could face them down from the higher +plane of hacker morality. And you could mock the police from the supercilious +heights of your hacker's quest for pure knowledge. + +But from the point of view of law enforcement and telco security, however, +Fry Guy was not really dangerous. The Atlanta Three WERE dangerous. +It wasn't the crimes they were committing, but the DANGER, +the potential hazard, the sheer TECHNICAL POWER LoD had accumulated, +that had made the situation untenable. Fry Guy was not LoD. +He'd never laid eyes on anyone in LoD; his only contacts with them +had been electronic. Core members of the Legion of Doom tended to meet +physically for conventions every year or so, to get drunk, give each other +the hacker high-sign, send out for pizza and ravage hotel suites. +Fry Guy had never done any of this. Deborah Daniels assessed Fry Guy +accurately as "an LoD wannabe." + +Nevertheless Fry Guy's crimes would be directly attributed to LoD +in much future police propaganda. LoD would be described as +"a closely knit group" involved in "numerous illegal activities" +including "stealing and modifying individual credit histories," +and "fraudulently obtaining money and property." Fry Guy did this, +but the Atlanta Three didn't; they simply weren't into theft, +but rather intrusion. This caused a strange kink in +the prosecution's strategy. LoD were accused of +"disseminating information about attacking computers +to other computer hackers in an effort to shift the focus +of law enforcement to those other hackers and away from the Legion of Doom." + +This last accusation (taken directly from a press release by the Chicago +Computer Fraud and Abuse Task Force) sounds particularly far-fetched. +One might conclude at this point that investigators would have been +well-advised to go ahead and "shift their focus" from the "Legion of Doom." +Maybe they SHOULD concentrate on "those other hackers"--the ones who were +actually stealing money and physical objects. + +But the Hacker Crackdown of 1990 was not a simple policing action. +It wasn't meant just to walk the beat in cyberspace--it was a CRACKDOWN, +a deliberate attempt to nail the core of the operation, to send a dire +and potent message that would settle the hash of the digital underground +for good. + +By this reasoning, Fry Guy wasn't much more than the electronic equivalent +of a cheap streetcorner dope dealer. As long as the masterminds of LoD were +still flagrantly operating, pushing their mountains of illicit knowledge +right and left, and whipping up enthusiasm for blatant lawbreaking, +then there would be an INFINITE SUPPLY of Fry Guys. + +Because LoD were flagrant, they had left trails everywhere, +to be picked up by law enforcement in New York, Indiana, +Florida, Texas, Arizona, Missouri, even Australia. +But 1990's war on the Legion of Doom was led out of Illinois, +by the Chicago Computer Fraud and Abuse Task Force. + +# + +The Computer Fraud and Abuse Task Force, led by federal prosecutor +William J. Cook, had started in 1987 and had swiftly become one +of the most aggressive local "dedicated computer-crime units." +Chicago was a natural home for such a group. The world's first +computer bulletin-board system had been invented in Illinois. +The state of Illinois had some of the nation's first and sternest +computer crime laws. Illinois State Police were markedly alert +to the possibilities of white-collar crime and electronic fraud. + +And William J. Cook in particular was a rising star in +electronic crime-busting. He and his fellow federal prosecutors +at the U.S. Attorney's office in Chicago had a tight relation +with the Secret Service, especially go-getting Chicago-based agent +Timothy Foley. While Cook and his Department of Justice colleagues +plotted strategy, Foley was their man on the street. + +Throughout the 1980s, the federal government had given prosecutors +an armory of new, untried legal tools against computer crime. +Cook and his colleagues were pioneers in the use of these new statutes +in the real-life cut-and-thrust of the federal courtroom. + +On October 2, 1986, the US Senate had passed the +"Computer Fraud and Abuse Act" unanimously, but there +were pitifully few convictions under this statute. +Cook's group took their name from this statute, +since they were determined to transform this powerful but +rather theoretical Act of Congress into a real-life engine +of legal destruction against computer fraudsters and scofflaws. + +It was not a question of merely discovering crimes, +investigating them, and then trying and punishing their +perpetrators. The Chicago unit, like most everyone else +in the business, already KNEW who the bad guys were: +the Legion of Doom and the writers and editors of Phrack. +The task at hand was to find some legal means of putting +these characters away. + +This approach might seem a bit dubious, to someone not +acquainted with the gritty realities of prosecutorial work. +But prosecutors don't put people in jail for crimes +they have committed; they put people in jail for crimes +they have committed THAT CAN BE PROVED IN COURT. +Chicago federal police put Al Capone in prison +for income-tax fraud. Chicago is a big town, +with a rough-and-ready bare-knuckle tradition +on both sides of the law. + +Fry Guy had broken the case wide open and alerted telco security +to the scope of the problem. But Fry Guy's crimes would not +put the Atlanta Three behind bars--much less the wacko underground +journalists of Phrack. So on July 22, 1989, the same day that +Fry Guy was raided in Indiana, the Secret Service descended upon +the Atlanta Three. + +This was likely inevitable. By the summer of 1989, law enforcement +were closing in on the Atlanta Three from at least six directions at once. +First, there were the leads from Fry Guy, which had led to the DNR registers +being installed on the lines of the Atlanta Three. The DNR evidence alone +would have finished them off, sooner or later. + +But second, the Atlanta lads were already well-known to Control-C +and his telco security sponsors. LoD's contacts with telco security +had made them overconfident and even more boastful than usual; +they felt that they had powerful friends in high places, +and that they were being openly tolerated by telco security. +But BellSouth's Intrusion Task Force were hot on the trail of LoD +and sparing no effort or expense. + +The Atlanta Three had also been identified by name and listed +on the extensive anti-hacker files maintained, and retailed for pay, +by private security operative John Maxfield of Detroit. +Maxfield, who had extensive ties to telco security +and many informants in the underground, was a bete noire +of the Phrack crowd, and the dislike was mutual. + + +The Atlanta Three themselves had written articles for Phrack. +This boastful act could not possibly escape telco and law enforcement +attention. + +"Knightmare," a high-school age hacker from Arizona, +was a close friend and disciple of Atlanta LoD, +but he had been nabbed by the formidable Arizona +Organized Crime and Racketeering Unit. Knightmare was +on some of LoD's favorite boards--"Black Ice" in particular-- +and was privy to their secrets. And to have Gail Thackeray, +the Assistant Attorney General of Arizona, on one's trail +was a dreadful peril for any hacker. + +And perhaps worst of all, Prophet had committed a major blunder +by passing an illicitly copied BellSouth computer-file to Knight Lightning, +who had published it in Phrack. This, as we will see, was an act of dire +consequence for almost everyone concerned. + +On July 22, 1989, the Secret Service showed up at the Leftist's house, +where he lived with his parents. A massive squad of some twenty officers +surrounded the building: Secret Service, federal marshals, local police, +possibly BellSouth telco security; it was hard to tell in the crush. +Leftist's dad, at work in his basement office, first noticed +a muscular stranger in plain clothes crashing through the +back yard with a drawn pistol. As more strangers poured +into the house, Leftist's dad naturally assumed there was +an armed robbery in progress. + +Like most hacker parents, Leftist's mom and dad had only the vaguest +notions of what their son had been up to all this time. Leftist had +a day-job repairing computer hardware. His obsession with computers +seemed a bit odd, but harmless enough, and likely to produce a well- +paying career. The sudden, overwhelming raid left Leftist's +parents traumatized. + +The Leftist himself had been out after work with his co-workers, +surrounding a couple of pitchers of margaritas. As he came trucking +on tequila-numbed feet up the pavement, toting a bag full of floppy-disks, +he noticed a large number of unmarked cars parked in his driveway. +All the cars sported tiny microwave antennas. + +The Secret Service had knocked the front door off its hinges, +almost flattening his mom. + +Inside, Leftist was greeted by Special Agent James Cool +of the US Secret Service, Atlanta office. Leftist was flabbergasted. +He'd never met a Secret Service agent before. He could not imagine +that he'd ever done anything worthy of federal attention. +He'd always figured that if his activities became intolerable, +one of his contacts in telco security would give him a private +phone-call and tell him to knock it off. + +But now Leftist was pat-searched for weapons by grim professionals, +and his bag of floppies was quickly seized. He and his parents were +all shepherded into separate rooms and grilled at length as a score +of officers scoured their home for anything electronic. + +Leftist was horrified as his treasured IBM AT personal computer +with its forty-meg hard disk, and his recently purchased 80386 IBM-clone +with a whopping hundred-meg hard disk, both went swiftly out the door +in Secret Service custody. They also seized all his disks, all his notebooks, +and a tremendous booty in dogeared telco documents that Leftist had snitched +out of trash dumpsters. + +Leftist figured the whole thing for a big misunderstanding. +He'd never been into MILITARY computers. He wasn't a SPY or a COMMUNIST. +He was just a good ol' Georgia hacker, and now he just wanted all these +people out of the house. But it seemed they wouldn't go until he made +some kind of statement. + +And so, he levelled with them. + +And that, Leftist said later from his federal prison camp in Talladega, +Alabama, was a big mistake. The Atlanta area was unique, +in that it had three members of the Legion of Doom who actually +occupied more or less the same physical locality. Unlike the rest +of LoD, who tended to associate by phone and computer, +Atlanta LoD actually WERE "tightly knit." It was no real +surprise that the Secret Service agents apprehending Urvile +at the computer-labs at Georgia Tech, would discover Prophet +with him as well. + +Urvile, a 21-year-old Georgia Tech student in polymer chemistry, +posed quite a puzzling case for law enforcement. Urvile--also known +as "Necron 99," as well as other handles, for he tended to change his +cover-alias about once a month--was both an accomplished hacker +and a fanatic simulation-gamer. + +Simulation games are an unusual hobby; but then hackers are unusual people, +and their favorite pastimes tend to be somewhat out of the ordinary. +The best-known American simulation game is probably "Dungeons & Dragons," +a multi-player parlor entertainment played with paper, maps, pencils, +statistical tables and a variety of oddly-shaped dice. Players pretend +to be heroic characters exploring a wholly-invented fantasy world. +The fantasy worlds of simulation gaming are commonly pseudo-medieval, +involving swords and sorcery--spell-casting wizards, knights in armor, +unicorns and dragons, demons and goblins. + +Urvile and his fellow gamers preferred their fantasies highly technological. +They made use of a game known as "G.U.R.P.S.," the "Generic Universal Role +Playing System," published by a company called Steve Jackson Games (SJG). + +"G.U.R.P.S." served as a framework for creating a wide variety of artificial +fantasy worlds. Steve Jackson Games published a smorgasboard of books, +full of detailed information and gaming hints, which were used to flesh-out +many different fantastic backgrounds for the basic GURPS framework. +Urvile made extensive use of two SJG books called GURPS High-Tech +and GURPS Special Ops. + +In the artificial fantasy-world of GURPS Special Ops, +players entered a modern fantasy of intrigue and international espionage. +On beginning the game, players started small and powerless, +perhaps as minor-league CIA agents or penny-ante arms dealers. +But as players persisted through a series of game sessions +(game sessions generally lasted for hours, over long, +elaborate campaigns that might be pursued for months on end) +then they would achieve new skills, new knowledge, new power. +They would acquire and hone new abilities, such as marksmanship, +karate, wiretapping, or Watergate burglary. They could also win +various kinds of imaginary booty, like Berettas, or martini shakers, +or fast cars with ejection seats and machine-guns under the headlights. + +As might be imagined from the complexity of these games, +Urvile's gaming notes were very detailed and extensive. +Urvile was a "dungeon-master," inventing scenarios +for his fellow gamers, giant simulated adventure-puzzles +for his friends to unravel. Urvile's game notes covered +dozens of pages with all sorts of exotic lunacy, all about +ninja raids on Libya and break-ins on encrypted Red Chinese supercomputers. +His notes were written on scrap-paper and kept in loose-leaf binders. + +The handiest scrap paper around Urvile's college digs were the many pounds of +BellSouth printouts and documents that he had snitched out of telco dumpsters. +His notes were written on the back of misappropriated telco property. +Worse yet, the gaming notes were chaotically interspersed with Urvile's +hand-scrawled records involving ACTUAL COMPUTER INTRUSIONS that he +had committed. + +Not only was it next to impossible to tell Urvile's fantasy game-notes +from cyberspace "reality," but Urvile himself barely made this distinction. +It's no exaggeration to say that to Urvile it was ALL a game. Urvile was +very bright, highly imaginative, and quite careless of other people's notions +of propriety. His connection to "reality" was not something to which he paid +a great deal of attention. + +Hacking was a game for Urvile. It was an amusement he was carrying out, +it was something he was doing for fun. And Urvile was an obsessive young man. +He could no more stop hacking than he could stop in the middle of +a jigsaw puzzle, or stop in the middle of reading a Stephen Donaldson +fantasy trilogy. (The name "Urvile" came from a best-selling Donaldson novel.) + +Urvile's airy, bulletproof attitude seriously annoyed his interrogators. +First of all, he didn't consider that he'd done anything wrong. +There was scarcely a shred of honest remorse in him. On the contrary, +he seemed privately convinced that his police interrogators were operating +in a demented fantasy-world all their own. Urvile was too polite +and well-behaved to say this straight-out, but his reactions were askew +and disquieting. + +For instance, there was the business about LoD's ability +to monitor phone-calls to the police and Secret Service. +Urvile agreed that this was quite possible, and posed +no big problem for LoD. In fact, he and his friends +had kicked the idea around on the "Black Ice" board, +much as they had discussed many other nifty notions, +such as building personal flame-throwers and jury-rigging +fistfulls of blasting-caps. They had hundreds of dial-up numbers +for government agencies that they'd gotten through scanning Atlanta phones, +or had pulled from raided VAX/VMS mainframe computers. + +Basically, they'd never gotten around to listening in on the cops +because the idea wasn't interesting enough to bother with. +Besides, if they'd been monitoring Secret Service phone calls, +obviously they'd never have been caught in the first place. Right? + +The Secret Service was less than satisfied with this rapier-like hacker logic. + +Then there was the issue of crashing the phone system. No problem, +Urvile admitted sunnily. Atlanta LoD could have shut down phone service +all over Atlanta any time they liked. EVEN THE 911 SERVICE? +Nothing special about that, Urvile explained patiently. +Bring the switch to its knees, with say the UNIX "makedir" bug, +and 911 goes down too as a matter of course. The 911 system +wasn't very interesting, frankly. It might be tremendously +interesting to cops (for odd reasons of their own), but as +technical challenges went, the 911 service was yawnsville. + +So of course the Atlanta Three could crash service. +They probably could have crashed service all over +BellSouth territory, if they'd worked at it for a while. +But Atlanta LoD weren't crashers. Only losers and rodents +were crashers. LoD were ELITE. + +Urvile was privately convinced that sheer technical +expertise could win him free of any kind of problem. +As far as he was concerned, elite status in the digital +underground had placed him permanently beyond the intellectual +grasp of cops and straights. Urvile had a lot to learn. + +Of the three LoD stalwarts, Prophet was in the most direct trouble. +Prophet was a UNIX programming expert who burrowed in and out +of the Internet as a matter of course. He'd started his hacking +career at around age 14, meddling with a UNIX mainframe system +at the University of North Carolina. + +Prophet himself had written the handy Legion of Doom +file "UNIX Use and Security From the Ground Up." +UNIX (pronounced "you-nicks") is a powerful, +flexible computer operating-system, for multi-user, +multi-tasking computers. In 1969, when UNIX was created +in Bell Labs, such computers were exclusive to large +corporations and universities, but today UNIX is run +on thousands of powerful home machines. UNIX was +particularly well-suited to telecommunications programming, +and had become a standard in the field. Naturally, UNIX +also became a standard for the elite hacker and phone phreak. +Lately, Prophet had not been so active as Leftist and Urvile, +but Prophet was a recidivist. In 1986, when he was eighteen, +Prophet had been convicted of "unauthorized access +to a computer network" in North Carolina. He'd been +discovered breaking into the Southern Bell Data Network, +a UNIX-based internal telco network supposedly closed to the public. +He'd gotten a typical hacker sentence: six months suspended, +120 hours community service, and three years' probation. + +After that humiliating bust, Prophet had gotten rid of most of his +tonnage of illicit phreak and hacker data, and had tried to go straight. +He was, after all, still on probation. But by the autumn of 1988, +the temptations of cyberspace had proved too much for young Prophet, +and he was shoulder-to-shoulder with Urvile and Leftist into some +of the hairiest systems around. + +In early September 1988, he'd broken into BellSouth's centralized +automation system, AIMSX or "Advanced Information Management System." +AIMSX was an internal business network for BellSouth, where telco +employees stored electronic mail, databases, memos, and calendars, +and did text processing. Since AIMSX did not have public dial-ups, +it was considered utterly invisible to the public, and was not well-secured +--it didn't even require passwords. Prophet abused an account known +as "waa1," the personal account of an unsuspecting telco employee. +Disguised as the owner of waa1, Prophet made about ten visits to AIMSX. + +Prophet did not damage or delete anything in the system. +His presence in AIMSX was harmless and almost invisible. +But he could not rest content with that. + +One particular piece of processed text on AIMSX was a telco document +known as "Bell South Standard Practice 660-225-104SV Control Office +Administration of Enhanced 911 Services for Special Services +and Major Account Centers dated March 1988." + +Prophet had not been looking for this document. It was merely one +among hundreds of similar documents with impenetrable titles. +However, having blundered over it in the course of his illicit +wanderings through AIMSX, he decided to take it with him as a trophy. +It might prove very useful in some future boasting, bragging, +and strutting session. So, some time in September 1988, +Prophet ordered the AIMSX mainframe computer to copy this document +(henceforth called simply called "the E911 Document") and to transfer +this copy to his home computer. + +No one noticed that Prophet had done this. He had "stolen" +the E911 Document in some sense, but notions of property +in cyberspace can be tricky. BellSouth noticed nothing wrong, +because BellSouth still had their original copy. They had not +been "robbed" of the document itself. Many people were supposed +to copy this document--specifically, people who worked for the +nineteen BellSouth "special services and major account centers," +scattered throughout the Southeastern United States. That was +what it was for, why it was present on a computer network +in the first place: so that it could be copied and read-- +by telco employees. But now the data had been copied +by someone who wasn't supposed to look at it. + +Prophet now had his trophy. But he further decided to store +yet another copy of the E911 Document on another person's computer. +This unwitting person was a computer enthusiast named Richard Andrews +who lived near Joliet, Illinois. Richard Andrews was a UNIX programmer +by trade, and ran a powerful UNIX board called "Jolnet," in the basement +of his house. + +Prophet, using the handle "Robert Johnson," had obtained an account +on Richard Andrews' computer. And there he stashed the E911 Document, +by storing it in his own private section of Andrews' computer. + +Why did Prophet do this? If Prophet had eliminated the E911 Document +from his own computer, and kept it hundreds of miles away, on another machine, under an +alias, then he might have been fairly safe from discovery and prosecution-- +although his sneaky action had certainly put the unsuspecting Richard Andrews +at risk. + +But, like most hackers, Prophet was a pack-rat for illicit data. +When it came to the crunch, he could not bear to part from his trophy. +When Prophet's place in Decatur, Georgia was raided in July 1989, +there was the E911 Document, a smoking gun. And there was Prophet +in the hands of the Secret Service, doing his best to "explain." + +Our story now takes us away from the Atlanta Three and their raids +of the Summer of 1989. We must leave Atlanta Three "cooperating fully" +with their numerous investigators. And all three of them did cooperate, +as their Sentencing Memorandum from the US District Court of the +Northern Division of Georgia explained--just before all three of them +were sentenced to various federal prisons in November 1990. + +We must now catch up on the other aspects of the war on the Legion of Doom. +The war on the Legion was a war on a network--in fact, a network of three +networks, which intertwined and interrelated in a complex fashion. +The Legion itself, with Atlanta LoD, and their hanger-on Fry Guy, +were the first network. The second network was Phrack magazine, +with its editors and contributors. + +The third network involved the electronic circle around a hacker +known as "Terminus." + +The war against these hacker networks was carried out by +a law enforcement network. Atlanta LoD and Fry Guy +were pursued by USSS agents and federal prosecutors in Atlanta, +Indiana, and Chicago. "Terminus" found himself pursued by USSS +and federal prosecutors from Baltimore and Chicago. And the war +against Phrack was almost entirely a Chicago operation. + +The investigation of Terminus involved a great deal of energy, +mostly from the Chicago Task Force, but it was to be the least-known +and least-publicized of the Crackdown operations. Terminus, who lived +in Maryland, was a UNIX programmer and consultant, fairly well-known +(under his given name) in the UNIX community, as an acknowledged expert +on AT&T minicomputers. Terminus idolized AT&T, especially Bellcore, +and longed for public recognition as a UNIX expert; his highest ambition +was to work for Bell Labs. + +But Terminus had odd friends and a spotted history. +Terminus had once been the subject of an admiring interview +in Phrack (Volume II, Issue 14, Phile 2--dated May 1987). +In this article, Phrack co-editor Taran King described +"Terminus" as an electronics engineer, 5'9", brown-haired, +born in 1959--at 28 years old, quite mature for a hacker. + +Terminus had once been sysop of a phreak/hack underground board +called "MetroNet," which ran on an Apple II. Later he'd replaced +"MetroNet" with an underground board called "MegaNet," +specializing in IBMs. In his younger days, Terminus had written +one of the very first and most elegant code-scanning programs +for the IBM-PC. This program had been widely distributed +in the underground. Uncounted legions of PC-owning phreaks and +hackers had used Terminus's scanner program to rip-off telco codes. +This feat had not escaped the attention of telco security; +it hardly could, since Terminus's earlier handle, "Terminal Technician," +was proudly written right on the program. + +When he became a full-time computer professional +(specializing in telecommunications programming), +he adopted the handle Terminus, meant to indicate that he +had "reached the final point of being a proficient hacker." +He'd moved up to the UNIX-based "Netsys" board on an AT&T computer, +with four phone lines and an impressive 240 megs of storage. +"Netsys" carried complete issues of Phrack, and Terminus was +quite friendly with its publishers, Taran King and Knight Lightning. + +In the early 1980s, Terminus had been a regular on Plovernet, +Pirate-80, Sherwood Forest and Shadowland, all well-known pirate boards, +all heavily frequented by the Legion of Doom. As it happened, Terminus +was never officially "in LoD," because he'd never been given the official +LoD high-sign and back-slap by Legion maven Lex Luthor. Terminus had +never physically met anyone from LoD. But that scarcely mattered much-- +the Atlanta Three themselves had never been officially vetted by Lex, either. + +As far as law enforcement was concerned, the issues were clear. +Terminus was a full-time, adult computer professional +with particular skills at AT&T software and hardware-- +but Terminus reeked of the Legion of Doom and the underground. + +On February 1, 1990--half a month after the Martin Luther King Day Crash-- +USSS agents Tim Foley from Chicago, and Jack Lewis from the Baltimore office, +accompanied by AT&T security officer Jerry Dalton, travelled to Middle Town, +Maryland. There they grilled Terminus in his home (to the stark terror of +his wife and small children), and, in their customary fashion, hauled his +computers out the door. + +The Netsys machine proved to contain a plethora of arcane UNIX software-- +proprietary source code formally owned by AT&T. Software such as: +UNIX System Five Release 3.2; UNIX SV Release 3.1; UUCP communications +software; KORN SHELL; RFS; IWB; WWB; DWB; the C++ programming language; +PMON; TOOL CHEST; QUEST; DACT, and S FIND. + +In the long-established piratical tradition of the underground, +Terminus had been trading this illicitly-copied software with +a small circle of fellow UNIX programmers. Very unwisely, +he had stored seven years of his electronic mail on his Netsys machine, +which documented all the friendly arrangements he had made with +his various colleagues. + +Terminus had not crashed the AT&T phone system on January 15. +He was, however, blithely running a not-for-profit AT&T +software-piracy ring. This was not an activity AT&T found amusing. +AT&T security officer Jerry Dalton valued this "stolen" property +at over three hundred thousand dollars. + +AT&T's entry into the tussle of free enterprise had been complicated +by the new, vague groundrules of the information economy. +Until the break-up of Ma Bell, AT&T was forbidden to sell +computer hardware or software. Ma Bell was the phone company; +Ma Bell was not allowed to use the enormous revenue from +telephone utilities, in order to finance any entry into +the computer market. + +AT&T nevertheless invented the UNIX operating system. +And somehow AT&T managed to make UNIX a minor source of income. +Weirdly, UNIX was not sold as computer software, +but actually retailed under an obscure regulatory +exemption allowing sales of surplus equipment and scrap. +Any bolder attempt to promote or retail UNIX would have +aroused angry legal opposition from computer companies. +Instead, UNIX was licensed to universities, at modest rates, +where the acids of academic freedom ate away steadily at AT&T's +proprietary rights. + +Come the breakup, AT&T recognized that UNIX was a potential gold-mine. +By now, large chunks of UNIX code had been created that were not AT&T's, +and were being sold by others. An entire rival UNIX-based operating system +had arisen in Berkeley, California (one of the world's great founts of +ideological hackerdom). Today, "hackers" commonly consider "Berkeley UNIX" +to be technically superior to AT&T's "System V UNIX," but AT&T has not +allowed mere technical elegance to intrude on the real-world business +of marketing proprietary software. AT&T has made its own code deliberately +incompatible with other folks' UNIX, and has written code that it can prove +is copyrightable, even if that code happens to be somewhat awkward--"kludgey." +AT&T UNIX user licenses are serious business agreements, replete with very +clear copyright statements and non-disclosure clauses. + +AT&T has not exactly kept the UNIX cat in the bag, +but it kept a grip on its scruff with some success. +By the rampant, explosive standards of software piracy, +AT&T UNIX source code is heavily copyrighted, well-guarded, +well-licensed. UNIX was traditionally run only on +mainframe machines, owned by large groups of suit-and-tie +professionals, rather than on bedroom machines where +people can get up to easy mischief. + +And AT&T UNIX source code is serious high-level programming. +The number of skilled UNIX programmers with any actual motive +to swipe UNIX source code is small. It's tiny, compared to +the tens of thousands prepared to rip-off, say, entertaining +PC games like "Leisure Suit Larry." + +But by 1989, the warez-d00d underground, in the persons of Terminus +and his friends, was gnawing at AT&T UNIX. And the property in question +was not sold for twenty bucks over the counter at the local branch of +Babbage's or Egghead's; this was massive, sophisticated, multi-line, +multi-author corporate code worth tens of thousands of dollars. + +It must be recognized at this point that Terminus's purported ring of UNIX +software pirates had not actually made any money from their suspected crimes. +The $300,000 dollar figure bandied about for the contents of Terminus's +computer did not mean that Terminus was in actual illicit possession +of three hundred thousand of AT&T's dollars. Terminus was shipping +software back and forth, privately, person to person, for free. +He was not making a commercial business of piracy. He hadn't +asked for money; he didn't take money. He lived quite modestly. + +AT&T employees--as well as freelance UNIX consultants, like Terminus-- +commonly worked with "proprietary" AT&T software, both in the office +and at home on their private machines. AT&T rarely sent security officers +out to comb the hard disks of its consultants. Cheap freelance UNIX +contractors were quite useful to AT&T; they didn't have health insurance +or retirement programs, much less union membership in the Communication +Workers of America. They were humble digital drudges, wandering with mop +and bucket through the Great Technological Temple of AT&T; but when the +Secret Service arrived at their homes, it seemed they were eating with +company silverware and sleeping on company sheets! Outrageously, they +behaved as if the things they worked with every day belonged to them! + +And these were no mere hacker teenagers with their hands full +of trash-paper and their noses pressed to the corporate windowpane. +These guys were UNIX wizards, not only carrying AT&T data in their +machines and their heads, but eagerly networking about it, +over machines that were far more powerful than anything previously +imagined in private hands. How do you keep people disposable, +yet assure their awestruck respect for your property? It was a dilemma. + +Much UNIX code was public-domain, available for free. Much "proprietary" +UNIX code had been extensively re-written, perhaps altered so much that it +became an entirely new product--or perhaps not. Intellectual property rights +for software developers were, and are, extraordinarily complex and confused. +And software "piracy," like the private copying of videos, is one of the most +widely practiced "crimes" in the world today. + +The USSS were not experts in UNIX or familiar with the customs of its use. +The United States Secret Service, considered as a body, did not have one single +person in it who could program in a UNIX environment--no, not even one. +The Secret Service WERE making extensive use of expert help, but the "experts" +they had chosen were AT&T and Bellcore security officials, the very victims of +the purported crimes under investigation, the very people whose interest in +AT&T's "proprietary" software was most pronounced. + +On February 6, 1990, Terminus was arrested by Agent Lewis. +Eventually, Terminus would be sent to prison for his illicit +use of a piece of AT&T software. + +The issue of pirated AT&T software would bubble along in the background +during the war on the Legion of Doom. Some half-dozen of Terminus's on-line +acquaintances, including people in Illinois, Texas and California, +were grilled by the Secret Service in connection with the illicit +copying of software. Except for Terminus, however, none were charged +with a crime. None of them shared his peculiar prominence in the +hacker underground. + +But that did not mean that these people would, or could, +stay out of trouble. The transferral of illicit data in +cyberspace is hazy and ill-defined business, with paradoxical +dangers for everyone concerned: hackers, signal carriers, +board owners, cops, prosecutors, even random passers-by. +Sometimes, well-meant attempts to avert trouble +or punish wrongdoing bring more trouble than +would simple ignorance, indifference or impropriety. + +Terminus's "Netsys" board was not a common-or-garden +bulletin board system, though it had most of the usual +functions of a board. Netsys was not a stand-alone machine, +but part of the globe-spanning "UUCP" cooperative network. +The UUCP network uses a set of Unix software programs called +"Unix-to-Unix Copy," which allows Unix systems to throw data to +one another at high speed through the public telephone network. +UUCP is a radically decentralized, not-for-profit network of UNIX computers. +There are tens of thousands of these UNIX machines. Some are small, +but many are powerful and also link to other networks. UUCP has +certain arcane links to major networks such as JANET, EasyNet, BITNET, +JUNET, VNET, DASnet, PeaceNet and FidoNet, as well as the gigantic Internet. +(The so-called "Internet" is not actually a network itself, but rather an +"internetwork" connections standard that allows several globe-spanning +computer networks to communicate with one another. Readers fascinated +by the weird and intricate tangles of modern computer networks may enjoy +John S. Quarterman's authoritative 719-page explication, The Matrix, +Digital Press, 1990.) + +A skilled user of Terminus' UNIX machine could send and receive +electronic mail from almost any major computer network in the world. +Netsys was not called a "board" per se, but rather a "node." +"Nodes" were larger, faster, and more sophisticated than mere "boards," +and for hackers, to hang out on internationally-connected "nodes" +was quite the step up from merely hanging out on local "boards." + +Terminus's Netsys node in Maryland had a number of direct +links to other, similar UUCP nodes, run by people who shared his +interests and at least something of his free-wheeling attitude. +One of these nodes was Jolnet, owned by Richard Andrews, who, +like Terminus, was an independent UNIX consultant. +Jolnet also ran UNIX, and could be contacted at high speed +by mainframe machines from all over the world. Jolnet was +quite a sophisticated piece of work, technically speaking, +but it was still run by an individual, as a private, +not-for-profit hobby. Jolnet was mostly used by other +UNIX programmers--for mail, storage, and access to networks. +Jolnet supplied access network access to about two hundred people, +as well as a local junior college. + +Among its various features and services, Jolnet also carried +Phrack magazine. + +For reasons of his own, Richard Andrews had become suspicious +of a new user called "Robert Johnson." Richard Andrews +took it upon himself to have a look at what "Robert Johnson" +was storing in Jolnet. And Andrews found the E911 Document. + +"Robert Johnson" was the Prophet from the Legion of Doom, +and the E911 Document was illicitly copied data from Prophet's +raid on the BellSouth computers. + +The E911 Document, a particularly illicit piece of digital property, +was about to resume its long, complex, and disastrous career. + +It struck Andrews as fishy that someone not a telephone employee +should have a document referring to the "Enhanced 911 System." +Besides, the document itself bore an obvious warning. + +"WARNING: NOT FOR USE OR DISCLOSURE OUTSIDE BELLSOUTH +OR ANY OF ITS SUBSIDIARIES EXCEPT UNDER WRITTEN AGREEMENT." + +These standard nondisclosure tags are often appended to all sorts +of corporate material. Telcos as a species are particularly notorious +for stamping most everything in sight as "not for use or disclosure." +Still, this particular piece of data was about the 911 System. +That sounded bad to Rich Andrews. + +Andrews was not prepared to ignore this sort of trouble. +He thought it would be wise to pass the document along +to a friend and acquaintance on the UNIX network, for consultation. +So, around September 1988, Andrews sent yet another copy of the +E911 Document electronically to an AT&T employee, one Charles Boykin, +who ran a UNIX-based node called "attctc" in Dallas, Texas. + +"Attctc" was the property of AT&T, and was run from AT&T's +Customer Technology Center in Dallas, hence the name "attctc." +"Attctc" was better-known as "Killer," the name of the machine +that the system was running on. "Killer" was a hefty, powerful, +AT&T 3B2 500 model, a multi-user, multi-tasking UNIX platform +with 32 meg of memory and a mind-boggling 3.2 Gigabytes of storage. +When Killer had first arrived in Texas, in 1985, the 3B2 had been +one of AT&T's great white hopes for going head-to-head with IBM +for the corporate computer-hardware market. "Killer" had been shipped +to the Customer Technology Center in the Dallas Infomart, essentially +a high-technology mall, and there it sat, a demonstration model. + +Charles Boykin, a veteran AT&T hardware and digital communications expert, +was a local technical backup man for the AT&T 3B2 system. As a display model +in the Infomart mall, "Killer" had little to do, and it seemed a shame +to waste the system's capacity. So Boykin ingeniously wrote some UNIX +bulletin-board software for "Killer," and plugged the machine in to the +local phone network. "Killer's" debut in late 1985 made it the first +publicly available UNIX site in the state of Texas. Anyone who wanted to +play was welcome. + +The machine immediately attracted an electronic community. +It joined the UUCP network, and offered network links +to over eighty other computer sites, all of which became dependent +on Killer for their links to the greater world of cyberspace. +And it wasn't just for the big guys; personal computer users +also stored freeware programs for the Amiga, the Apple, +the IBM and the Macintosh on Killer's vast 3,200 meg archives. +At one time, Killer had the largest library of public-domain +Macintosh software in Texas. + +Eventually, Killer attracted about 1,500 users, +all busily communicating, uploading and downloading, +getting mail, gossipping, and linking to arcane +and distant networks. + +Boykin received no pay for running Killer. He considered +it good publicity for the AT&T 3B2 system (whose sales were +somewhat less than stellar), but he also simply enjoyed +the vibrant community his skill had created. He gave away +the bulletin-board UNIX software he had written, free of charge. + +In the UNIX programming community, Charlie Boykin had the +reputation of a warm, open-hearted, level-headed kind of guy. +In 1989, a group of Texan UNIX professionals voted Boykin +"System Administrator of the Year." He was considered +a fellow you could trust for good advice. + +In September 1988, without warning, the E911 Document +came plunging into Boykin's life, forwarded by Richard Andrews. +Boykin immediately recognized that the Document was hot property. +He was not a voice-communications man, and knew little about +the ins and outs of the Baby Bells, but he certainly knew what +the 911 System was, and he was angry to see confidential data +about it in the hands of a nogoodnik. This was clearly a +matter for telco security. So, on September 21, 1988, Boykin +made yet ANOTHER copy of the E911 Document and passed this +one along to a professional acquaintance of his, one Jerome Dalton, +from AT&T Corporate Information Security. Jerry Dalton was the +very fellow who would later raid Terminus's house. + +From AT&T's security division, the E911 Document went to Bellcore. + +Bellcore (or BELL COmmunications REsearch) had once been the central +laboratory of the Bell System. Bell Labs employees had invented +the UNIX operating system. Now Bellcore was a quasi-independent, +jointly owned company that acted as the research arm for all seven +of the Baby Bell RBOCs. Bellcore was in a good position to co-ordinate +security technology and consultation for the RBOCs, and the gentleman in +charge of this effort was Henry M. Kluepfel, a veteran of the Bell System +who had worked there for twenty-four years. + +On October 13, 1988, Dalton passed the E911 Document to Henry Kluepfel. +Kluepfel, a veteran expert witness in telecommunications fraud and +computer-fraud cases, had certainly seen worse trouble than this. +He recognized the document for what it was: a trophy from a hacker break-in. + +However, whatever harm had been done in the intrusion was presumably old news. +At this point there seemed little to be done. Kluepfel made a careful note +of the circumstances and shelved the problem for the time being. + +Whole months passed. + +February 1989 arrived. The Atlanta Three were living it up +in Bell South's switches, and had not yet met their comeuppance. +The Legion was thriving. So was Phrack magazine. +A good six months had passed since Prophet's AIMSX break-in. +Prophet, as hackers will, grew weary of sitting on his laurels. +"Knight Lightning" and "Taran King," the editors of Phrack, +were always begging Prophet for material they could publish. +Prophet decided that the heat must be off by this time, +and that he could safely brag, boast, and strut. + +So he sent a copy of the E911 Document--yet another one-- +from Rich Andrews' Jolnet machine to Knight Lightning's +BITnet account at the University of Missouri. +Let's review the fate of the document so far. + +0. The original E911 Document. This in the AIMSX system +on a mainframe computer in Atlanta, available to hundreds of people, +but all of them, presumably, BellSouth employees. An unknown number +of them may have their own copies of this document, but they are all +professionals and all trusted by the phone company. + +1. Prophet's illicit copy, at home on his own computer in Decatur, Georgia. + +2. Prophet's back-up copy, stored on Rich Andrew's Jolnet machine + in the basement of Rich Andrews' house near Joliet Illinois. + +3. Charles Boykin's copy on "Killer" in Dallas, Texas, + sent by Rich Andrews from Joliet. + +4. Jerry Dalton's copy at AT&T Corporate Information Security in New Jersey, + sent from Charles Boykin in Dallas. + +5. Henry Kluepfel's copy at Bellcore security headquarters in New Jersey, + sent by Dalton. +6. Knight Lightning's copy, sent by Prophet from Rich Andrews' machine, + and now in Columbia, Missouri. + +We can see that the "security" situation of this proprietary document, +once dug out of AIMSX, swiftly became bizarre. Without any money +changing hands, without any particular special effort, this data +had been reproduced at least six times and had spread itself all over +the continent. By far the worst, however, was yet to come. + +In February 1989, Prophet and Knight Lightning bargained electronically +over the fate of this trophy. Prophet wanted to boast, but, at the same time, +scarcely wanted to be caught. + +For his part, Knight Lightning was eager to publish as much of the document +as he could manage. Knight Lightning was a fledgling political-science major +with a particular interest in freedom-of-information issues. He would gladly +publish most anything that would reflect glory on the prowess of the +underground and embarrass the telcos. However, Knight Lightning himself +had contacts in telco security, and sometimes consulted them on material +he'd received that might be too dicey for publication. + +Prophet and Knight Lightning decided to edit the E911 Document +so as to delete most of its identifying traits. First of all, +its large "NOT FOR USE OR DISCLOSURE" warning had to go. +Then there were other matters. For instance, it listed +the office telephone numbers of several BellSouth 911 +specialists in Florida. If these phone numbers were +published in Phrack, the BellSouth employees involved +would very likely be hassled by phone phreaks, +which would anger BellSouth no end, and pose a +definite operational hazard for both Prophet and Phrack. + +So Knight Lightning cut the Document almost in half, +removing the phone numbers and some of the touchier +and more specific information. He passed it back +electronically to Prophet; Prophet was still nervous, +so Knight Lightning cut a bit more. They finally agreed +that it was ready to go, and that it would be published +in Phrack under the pseudonym, "The Eavesdropper." + +And this was done on February 25, 1989. + +The twenty-fourth issue of Phrack featured a chatty interview +with co-ed phone-phreak "Chanda Leir," three articles on BITNET +and its links to other computer networks, an article on 800 and 900 +numbers by "Unknown User," "VaxCat's" article on telco basics +(slyly entitled "Lifting Ma Bell's Veil of Secrecy,)" and +the usual "Phrack World News." + +The News section, with painful irony, featured an extended account +of the sentencing of "Shadowhawk," an eighteen-year-old Chicago hacker +who had just been put in federal prison by William J. Cook himself. + +And then there were the two articles by "The Eavesdropper." +The first was the edited E911 Document, now titled +"Control Office Administration Of Enhanced 911 Services +for Special Services and Major Account Centers." +Eavesdropper's second article was a glossary of terms +explaining the blizzard of telco acronyms and buzzwords +in the E911 Document. + +The hapless document was now distributed, in the usual Phrack routine, +to a good one hundred and fifty sites. Not a hundred and fifty PEOPLE, +mind you--a hundred and fifty SITES, some of these sites linked to UNIX +nodes or bulletin board systems, which themselves had readerships of tens, +dozens, even hundreds of people. + +This was February 1989. Nothing happened immediately. +Summer came, and the Atlanta crew were raided by the Secret Service. +Fry Guy was apprehended. Still nothing whatever happened to Phrack. +Six more issues of Phrack came out, 30 in all, more or less on +a monthly schedule. Knight Lightning and co-editor Taran King +went untouched. + +Phrack tended to duck and cover whenever the heat came down. +During the summer busts of 1987--(hacker busts tended to cluster in summer, +perhaps because hackers were easier to find at home than in college)-- +Phrack had ceased publication for several months, and laid low. +Several LoD hangers-on had been arrested, but nothing had happened +to the Phrack crew, the premiere gossips of the underground. +In 1988, Phrack had been taken over by a new editor, +"Crimson Death," a raucous youngster with a taste for anarchy files. +1989, however, looked like a bounty year for the underground. +Knight Lightning and his co-editor Taran King took up the reins again, +and Phrack flourished throughout 1989. Atlanta LoD went down hard in +the summer of 1989, but Phrack rolled merrily on. Prophet's E911 Document +seemed unlikely to cause Phrack any trouble. By January 1990, +it had been available in Phrack for almost a year. Kluepfel and Dalton, +officers of Bellcore and AT&T security, had possessed the document +for sixteen months--in fact, they'd had it even before Knight Lightning +himself, and had done nothing in particular to stop its distribution. +They hadn't even told Rich Andrews or Charles Boykin to erase the copies +from their UNIX nodes, Jolnet and Killer. + +But then came the monster Martin Luther King Day Crash of January 15, 1990. + +A flat three days later, on January 18, four agents showed up +at Knight Lightning's fraternity house. One was Timothy Foley, +the second Barbara Golden, both of them Secret Service agents +from the Chicago office. Also along was a University of Missouri +security officer, and Reed Newlin, a security man from Southwestern Bell, +the RBOC having jurisdiction over Missouri. + +Foley accused Knight Lightning of causing the nationwide crash +of the phone system. + +Knight Lightning was aghast at this allegation. On the face of it, +the suspicion was not entirely implausible--though Knight Lightning +knew that he himself hadn't done it. Plenty of hot-dog hackers +had bragged that they could crash the phone system, however. +"Shadowhawk," for instance, the Chicago hacker whom William Cook +had recently put in jail, had several times boasted on boards +that he could "shut down AT&T's public switched network." + +And now this event, or something that looked just like it, +had actually taken place. The Crash had lit a fire under +the Chicago Task Force. And the former fence-sitters at +Bellcore and AT&T were now ready to roll. The consensus +among telco security--already horrified by the skill of +the BellSouth intruders --was that the digital underground +was out of hand. LoD and Phrack must go. And in publishing +Prophet's E911 Document, Phrack had provided law enforcement +with what appeared to be a powerful legal weapon. + +Foley confronted Knight Lightning about the E911 Document. + +Knight Lightning was cowed. He immediately began "cooperating fully" +in the usual tradition of the digital underground. + +He gave Foley a complete run of Phrack, printed out in a set +of three-ring binders. He handed over his electronic mailing list +of Phrack subscribers. Knight Lightning was grilled for four hours +by Foley and his cohorts. Knight Lightning admitted that Prophet +had passed him the E911 Document, and he admitted that he had known +it was stolen booty from a hacker raid on a telephone company. +Knight Lightning signed a statement to this effect, and agreed, +in writing, to cooperate with investigators. + +Next day--January 19, 1990, a Friday --the Secret Service returned +with a search warrant, and thoroughly searched Knight Lightning's +upstairs room in the fraternity house. They took all his floppy disks, +though, interestingly, they left Knight Lightning in possession +of both his computer and his modem. (The computer had no hard disk, +and in Foley's judgement was not a store of evidence.) But this was a +very minor bright spot among Knight Lightning's rapidly multiplying troubles. +By this time, Knight Lightning was in plenty of hot water, not only with +federal police, prosecutors, telco investigators, and university security, +but with the elders of his own campus fraternity, who were outraged +to think that they had been unwittingly harboring a federal computer-criminal. + +On Monday, Knight Lightning was summoned to Chicago, where he was +further grilled by Foley and USSS veteran agent Barbara Golden, this time +with an attorney present. And on Tuesday, he was formally indicted +by a federal grand jury. + +The trial of Knight Lightning, which occurred on July 24-27, 1990, +was the crucial show-trial of the Hacker Crackdown. We will examine +the trial at some length in Part Four of this book. + +In the meantime, we must continue our dogged pursuit of the E911 Document. + +It must have been clear by January 1990 that the E911 Document, +in the form Phrack had published it back in February 1989, +had gone off at the speed of light in at least a hundred +and fifty different directions. To attempt to put this +electronic genie back in the bottle was flatly impossible. + +And yet, the E911 Document was STILL stolen property, +formally and legally speaking. Any electronic transference +of this document, by anyone unauthorized to have it, +could be interpreted as an act of wire fraud. Interstate +transfer of stolen property, including electronic property, +was a federal crime. + +The Chicago Computer Fraud and Abuse Task Force had been assured +that the E911 Document was worth a hefty sum of money. In fact, +they had a precise estimate of its worth from BellSouth security personnel: +$79,449. A sum of this scale seemed to warrant vigorous prosecution. +Even if the damage could not be undone, at least this large sum +offered a good legal pretext for stern punishment of the thieves. +It seemed likely to impress judges and juries. And it could be used +in court to mop up the Legion of Doom. + +The Atlanta crowd was already in the bag, by the time +the Chicago Task Force had gotten around to Phrack. +But the Legion was a hydra-headed thing. In late 89, +a brand-new Legion of Doom board, "Phoenix Project," +had gone up in Austin, Texas. Phoenix Project was sysoped +by no less a man than the Mentor himself, ably assisted by +University of Texas student and hardened Doomster "Erik Bloodaxe." + +As we have seen from his Phrack manifesto, the Mentor was a hacker +zealot who regarded computer intrusion as something close to a moral duty. +Phoenix Project was an ambitious effort, intended to revive the digital +underground to what Mentor considered the full flower of the early 80s. +The Phoenix board would also boldly bring elite hackers face-to-face +with the telco "opposition." On "Phoenix," America's cleverest hackers +would supposedly shame the telco squareheads out of their stick-in-the-mud +attitudes, and perhaps convince them that the Legion of Doom elite were really +an all-right crew. The premiere of "Phoenix Project" was heavily trumpeted +by Phrack,and "Phoenix Project" carried a complete run of Phrack issues, +including the E911 Document as Phrack had published it. + +Phoenix Project was only one of many--possibly hundreds--of nodes and boards +all over America that were in guilty possession of the E911 Document. +But Phoenix was an outright, unashamed Legion of Doom board. +Under Mentor's guidance, it was flaunting itself in the face +of telco security personnel. Worse yet, it was actively trying +to WIN THEM OVER as sympathizers for the digital underground elite. +"Phoenix" had no cards or codes on it. Its hacker elite considered +Phoenix at least technically legal. But Phoenix was a corrupting influence, +where hacker anarchy was eating away like digital acid at the underbelly +of corporate propriety. + +The Chicago Computer Fraud and Abuse Task Force now prepared +to descend upon Austin, Texas. + +Oddly, not one but TWO trails of the Task Force's investigation led +toward Austin. The city of Austin, like Atlanta, had made itself +a bulwark of the Sunbelt's Information Age, with a strong university +research presence, and a number of cutting-edge electronics companies, +including Motorola, Dell, CompuAdd, IBM, Sematech and MCC. + +Where computing machinery went, hackers generally followed. +Austin boasted not only "Phoenix Project," currently LoD's +most flagrant underground board, but a number of UNIX nodes. + +One of these nodes was "Elephant," run by a UNIX consultant +named Robert Izenberg. Izenberg, in search of a relaxed Southern +lifestyle and a lowered cost-of-living, had recently migrated +to Austin from New Jersey. In New Jersey, Izenberg had worked +for an independent contracting company, programming UNIX code for +AT&T itself. "Terminus" had been a frequent user on Izenberg's +privately owned Elephant node. + +Having interviewed Terminus and examined the records on Netsys, +the Chicago Task Force were now convinced that they had discovered +an underground gang of UNIX software pirates, who were demonstrably +guilty of interstate trafficking in illicitly copied AT&T source code. +Izenberg was swept into the dragnet around Terminus, the self-proclaimed +ultimate UNIX hacker. + +Izenberg, in Austin, had settled down into a UNIX job +with a Texan branch of IBM. Izenberg was no longer +working as a contractor for AT&T, but he had friends +in New Jersey, and he still logged on to AT&T UNIX +computers back in New Jersey, more or less whenever +it pleased him. Izenberg's activities appeared highly +suspicious to the Task Force. Izenberg might well be +breaking into AT&T computers, swiping AT&T software, +and passing it to Terminus and other possible confederates, +through the UNIX node network. And this data was worth, +not merely $79,499, but hundreds of thousands of dollars! + +On February 21, 1990, Robert Izenberg arrived home +from work at IBM to find that all the computers +had mysteriously vanished from his Austin apartment. +Naturally he assumed that he had been robbed. +His "Elephant" node, his other machines, his notebooks, +his disks, his tapes, all gone! However, nothing much +else seemed disturbed--the place had not been ransacked. +The puzzle becaming much stranger some five minutes later. +Austin U. S. Secret Service Agent Al Soliz, accompanied by +University of Texas campus-security officer Larry Coutorie +and the ubiquitous Tim Foley, made their appearance at Izenberg's door. +They were in plain clothes: slacks, polo shirts. They came in, +and Tim Foley accused Izenberg of belonging to the Legion of Doom. + +Izenberg told them that he had never heard of the "Legion of Doom." +And what about a certain stolen E911 Document, that posed a direct +threat to the police emergency lines? Izenberg claimed that he'd +never heard of that, either. + +His interrogators found this difficult to believe. +Didn't he know Terminus? + +Who? + +They gave him Terminus's real name. Oh yes, said Izenberg. +He knew THAT guy all right--he was leading discussions +on the Internet about AT&T computers, especially the AT&T 3B2. + +AT&T had thrust this machine into the marketplace, +but, like many of AT&T's ambitious attempts to enter +the computing arena, the 3B2 project had something less +than a glittering success. Izenberg himself had been +a contractor for the division of AT&T that supported the 3B2. +The entire division had been shut down. + +Nowadays, the cheapest and quickest way to get help with this +fractious piece of machinery was to join one of Terminus's +discussion groups on the Internet, where friendly and knowledgeable +hackers would help you for free. Naturally the remarks within this +group were less than flattering about the Death Star. . .was +THAT the problem? + +Foley told Izenberg that Terminus had been acquiring hot software +through his, Izenberg's, machine. + +Izenberg shrugged this off. A good eight megabytes of data flowed +through his UUCP site every day. UUCP nodes spewed data like fire hoses. +Elephant had been directly linked to Netsys--not surprising, since Terminus +was a 3B2 expert and Izenberg had been a 3B2 contractor. +Izenberg was also linked to "attctc" and the University of Texas. +Terminus was a well-known UNIX expert, and might have been up to +all manner of hijinks on Elephant. Nothing Izenberg could do about that. +That was physically impossible. Needle in a haystack. + +In a four-hour grilling, Foley urged Izenberg to come clean +and admit that he was in conspiracy with Terminus, +and a member of the Legion of Doom. + +Izenberg denied this. He was no weirdo teenage hacker-- +he was thirty-two years old, and didn't even have a "handle." +Izenberg was a former TV technician and electronics specialist +who had drifted into UNIX consulting as a full-grown adult. +Izenberg had never met Terminus, physically. He'd once bought +a cheap high-speed modem from him, though. + +Foley told him that this modem (a Telenet T2500 which ran at 19.2 kilobaud, +and which had just gone out Izenberg's door in Secret Service custody) +was likely hot property. Izenberg was taken aback to hear this; but then +again, most of Izenberg's equipment, like that of most freelance professionals +in the industry, was discounted, passed hand-to-hand through various kinds +of barter and gray-market. There was no proof that the modem was stolen, +and even if it were, Izenberg hardly saw how that gave them the right +to take every electronic item in his house. + +Still, if the United States Secret Service figured they needed +his computer for national security reasons--or whatever-- +then Izenberg would not kick. He figured he would somehow +make the sacrifice of his twenty thousand dollars' worth +of professional equipment, in the spirit of full cooperation +and good citizenship. + +Robert Izenberg was not arrested. Izenberg was not charged with any crime. +His UUCP node--full of some 140 megabytes of the files, mail, and data +of himself and his dozen or so entirely innocent users--went out the door +as "evidence." Along with the disks and tapes, Izenberg had lost about +800 megabytes of data. + +Six months would pass before Izenberg decided to phone the Secret Service +and ask how the case was going. That was the first time that Robert Izenberg +would ever hear the name of William Cook. As of January 1992, a full +two years after the seizure, Izenberg, still not charged with any crime, +would be struggling through the morass of the courts, in hope of recovering +his thousands of dollars' worth of seized equipment. + +In the meantime, the Izenberg case received absolutely no press coverage. +The Secret Service had walked into an Austin home, removed a UNIX bulletin- +board system, and met with no operational difficulties whatsoever. + +Except that word of a crackdown had percolated through the Legion of Doom. +"The Mentor" voluntarily shut down "The Phoenix Project." It seemed a pity, +especially as telco security employees had, in fact, shown up on Phoenix, +just as he had hoped--along with the usual motley crowd of LoD heavies, +hangers-on, phreaks, hackers and wannabes. There was "Sandy" Sandquist from +US SPRINT security, and some guy named Henry Kluepfel, from Bellcore itself! +Kluepfel had been trading friendly banter with hackers on Phoenix since +January 30th (two weeks after the Martin Luther King Day Crash). +The presence of such a stellar telco official seemed quite the coup +for Phoenix Project. + +Still, Mentor could judge the climate. Atlanta in ruins, +Phrack in deep trouble, something weird going on with UNIX nodes-- +discretion was advisable. Phoenix Project went off-line. + +Kluepfel, of course, had been monitoring this LoD bulletin +board for his own purposes--and those of the Chicago unit. +As far back as June 1987, Kluepfel had logged on to a Texas +underground board called "Phreak Klass 2600." There he'd +discovered an Chicago youngster named "Shadowhawk," +strutting and boasting about rifling AT&T computer files, +and bragging of his ambitions to riddle AT&T's Bellcore +computers with trojan horse programs. Kluepfel had passed +the news to Cook in Chicago, Shadowhawk's computers +had gone out the door in Secret Service custody, +and Shadowhawk himself had gone to jail. + +Now it was Phoenix Project's turn. Phoenix Project postured +about "legality" and "merely intellectual interest," but it reeked +of the underground. It had Phrack on it. It had the E911 Document. +It had a lot of dicey talk about breaking into systems, including some +bold and reckless stuff about a supposed "decryption service" that Mentor +and friends were planning to run, to help crack encrypted passwords off +of hacked systems. + +Mentor was an adult. There was a bulletin board at his place of work, +as well. Kleupfel logged onto this board, too, and discovered it to be +called "Illuminati." It was run by some company called Steve Jackson Games. + +On March 1, 1990, the Austin crackdown went into high gear. + +On the morning of March 1--a Thursday--21-year-old University of Texas +student "Erik Bloodaxe," co-sysop of Phoenix Project and an avowed member +of the Legion of Doom, was wakened by a police revolver levelled at his head. + +Bloodaxe watched, jittery, as Secret Service agents +appropriated his 300 baud terminal and, rifling his files, +discovered his treasured source-code for Robert Morris's +notorious Internet Worm. But Bloodaxe, a wily operator, +had suspected that something of the like might be coming. +All his best equipment had been hidden away elsewhere. +The raiders took everything electronic, however, +including his telephone. They were stymied by his +hefty arcade-style Pac-Man game, and left it in place, +as it was simply too heavy to move. + +Bloodaxe was not arrested. He was not charged with any crime. +A good two years later, the police still had what they had +taken from him, however. + +The Mentor was less wary. The dawn raid rousted him and his wife +from bed in their underwear, and six Secret Service agents, +accompanied by an Austin policeman and Henry Kluepfel himself, +made a rich haul. Off went the works, into the agents' white +Chevrolet minivan: an IBM PC-AT clone with 4 meg of RAM and +a 120-meg hard disk; a Hewlett-Packard LaserJet II printer; +a completely legitimate and highly expensive SCO-Xenix 286 +operating system; Pagemaker disks and documentation; +and the Microsoft Word word-processing program. Mentor's wife +had her incomplete academic thesis stored on the hard-disk; +that went, too, and so did the couple's telephone. As of two years later, +all this property remained in police custody. + +Mentor remained under guard in his apartment as agents prepared +to raid Steve Jackson Games. The fact that this was a business +headquarters and not a private residence did not deter the agents. +It was still very early; no one was at work yet. The agents prepared +to break down the door, but Mentor, eavesdropping on the Secret Service +walkie-talkie traffic, begged them not to do it, and offered his key +to the building. + +The exact details of the next events are unclear. The agents +would not let anyone else into the building. Their search warrant, +when produced, was unsigned. Apparently they breakfasted from the local +"Whataburger," as the litter from hamburgers was later found inside. +They also extensively sampled a bag of jellybeans kept by an SJG employee. +Someone tore a "Dukakis for President" sticker from the wall. + +SJG employees, diligently showing up for the day's work, were met +at the door and briefly questioned by U.S. Secret Service agents. +The employees watched in astonishment as agents wielding crowbars +and screwdrivers emerged with captive machines. They attacked +outdoor storage units with boltcutters. The agents wore +blue nylon windbreakers with "SECRET SERVICE" stencilled +across the back, with running-shoes and jeans. + +Jackson's company lost three computers, several hard-disks, +hundred of floppy disks, two monitors, three modems, +a laser printer, various powercords, cables, and adapters +(and, oddly, a small bag of screws, bolts and nuts). +The seizure of Illuminati BBS deprived SJG of all the programs, +text files, and private e-mail on the board. The loss of two other +SJG computers was a severe blow as well, since it caused the loss +of electronically stored contracts, financial projections, +address directories, mailing lists, personnel files, +business correspondence, and, not least, the drafts +of forthcoming games and gaming books. + +No one at Steve Jackson Games was arrested. No one was accused +of any crime. No charges were filed. Everything appropriated +was officially kept as "evidence" of crimes never specified. + +After the Phrack show-trial, the Steve Jackson Games scandal +was the most bizarre and aggravating incident of the Hacker +Crackdown of 1990. This raid by the Chicago Task Force +on a science-fiction gaming publisher was to rouse a +swarming host of civil liberties issues, and gave rise +to an enduring controversy that was still re-complicating itself, +and growing in the scope of its implications, a full two years later. + +The pursuit of the E911 Document stopped with the Steve Jackson Games raid. +As we have seen, there were hundreds, perhaps thousands of computer users +in America with the E911 Document in their possession. Theoretically, +Chicago had a perfect legal right to raid any of these people, +and could have legally seized the machines of anybody who subscribed to Phrack. +However, there was no copy of the E911 Document on Jackson's Illuminati board. +And there the Chicago raiders stopped dead; they have not raided anyone since. + +It might be assumed that Rich Andrews and Charlie Boykin, who had brought +the E911 Document to the attention of telco security, might be spared +any official suspicion. But as we have seen, the willingness to +"cooperate fully" offers little, if any, assurance against federal +anti-hacker prosecution. + +Richard Andrews found himself in deep trouble, thanks to the E911 Document. +Andrews lived in Illinois, the native stomping grounds of the Chicago +Task Force. On February 3 and 6, both his home and his place of work +were raided by USSS. His machines went out the door, too, and he was +grilled at length (though not arrested). Andrews proved to be in +purportedly guilty possession of: UNIX SVR 3.2; UNIX SVR 3.1; UUCP; +PMON; WWB; IWB; DWB; NROFF; KORN SHELL '88; C++; and QUEST, +among other items. Andrews had received this proprietary code-- +which AT&T officially valued at well over $250,000--through the +UNIX network, much of it supplied to him as a personal favor by Terminus. +Perhaps worse yet, Andrews admitted to returning the favor, by passing +Terminus a copy of AT&T proprietary STARLAN source code. + +Even Charles Boykin, himself an AT&T employee, entered some very hot water. +By 1990, he'd almost forgotten about the E911 problem he'd reported in +September 88; in fact, since that date, he'd passed two more security alerts +to Jerry Dalton, concerning matters that Boykin considered far worse than +the E911 Document. + +But by 1990, year of the crackdown, AT&T Corporate Information Security +was fed up with "Killer." This machine offered no direct income to AT&T, +and was providing aid and comfort to a cloud of suspicious yokels +from outside the company, some of them actively malicious toward AT&T, +its property, and its corporate interests. Whatever goodwill and publicity +had been won among Killer's 1,500 devoted users was considered no longer +worth the security risk. On February 20, 1990, Jerry Dalton arrived in +Dallas and simply unplugged the phone jacks, to the puzzled alarm +of Killer's many Texan users. Killer went permanently off-line, +with the loss of vast archives of programs and huge quantities +of electronic mail; it was never restored to service. AT&T showed +no particular regard for the "property" of these 1,500 people. +Whatever "property" the users had been storing on AT&T's computer +simply vanished completely. + +Boykin, who had himself reported the E911 problem, +now found himself under a cloud of suspicion. In a weird +private-security replay of the Secret Service seizures, +Boykin's own home was visited by AT&T Security and his +own machines were carried out the door. + +However, there were marked special features in the Boykin case. +Boykin's disks and his personal computers were swiftly examined +by his corporate employers and returned politely in just two days-- +(unlike Secret Service seizures, which commonly take months or years). +Boykin was not charged with any crime or wrongdoing, and he kept his job +with AT&T (though he did retire from AT&T in September 1991, +at the age of 52). + +It's interesting to note that the US Secret Service somehow failed +to seize Boykin's "Killer" node and carry AT&T's own computer out the door. +Nor did they raid Boykin's home. They seemed perfectly willing to take the +word of AT&T Security that AT&T's employee, and AT&T's "Killer" node, +were free of hacker contraband and on the up-and-up. + +It's digital water-under-the-bridge at this point, as Killer's +3,200 megabytes of Texan electronic community were erased in 1990, +and "Killer" itself was shipped out of the state. + +But the experiences of Andrews and Boykin, and the users of their systems, +remained side issues. They did not begin to assume the social, political, +and legal importance that gathered, slowly but inexorably, around the issue +of the raid on Steve Jackson Games. + +# + +We must now turn our attention to Steve Jackson Games itself, +and explain what SJG was, what it really did, and how it had +managed to attract this particularly odd and virulent kind of trouble. +The reader may recall that this is not the first but the second time +that the company has appeared in this narrative; a Steve Jackson game +called GURPS was a favorite pastime of Atlanta hacker Urvile, +and Urvile's science-fictional gaming notes had been mixed up +promiscuously with notes about his actual computer intrusions. + +First, Steve Jackson Games, Inc., was NOT a publisher of "computer games." +SJG published "simulation games," parlor games that were played on paper, +with pencils, and dice, and printed guidebooks full of rules and +statistics tables. There were no computers involved in the games themselves. +When you bought a Steve Jackson Game, you did not receive any software disks. +What you got was a plastic bag with some cardboard game tokens, +maybe a few maps or a deck of cards. Most of their products were books. + +However, computers WERE deeply involved in the Steve Jackson Games business. +Like almost all modern publishers, Steve Jackson and his fifteen employees +used computers to write text, to keep accounts, and to run the business +generally. They also used a computer to run their official bulletin board +system for Steve Jackson Games, a board called Illuminati. On Illuminati, +simulation gamers who happened to own computers and modems could associate, +trade mail, debate the theory and practice of gaming, and keep up with the +company's news and its product announcements. + +Illuminati was a modestly popular board, run on a small computer +with limited storage, only one phone-line, and no ties to large-scale +computer networks. It did, however, have hundreds of users, +many of them dedicated gamers willing to call from out-of-state. + +Illuminati was NOT an "underground" board. It did not feature hints +on computer intrusion, or "anarchy files," or illicitly posted +credit card numbers, or long-distance access codes. +Some of Illuminati's users, however, were members of the Legion of Doom. +And so was one of Steve Jackson's senior employees--the Mentor. +The Mentor wrote for Phrack, and also ran an underground board, +Phoenix Project--but the Mentor was not a computer professional. +The Mentor was the managing editor of Steve Jackson Games and +a professional game designer by trade. These LoD members did not +use Illuminati to help their HACKING activities. They used it to +help their GAME-PLAYING activities--and they were even more dedicated +to simulation gaming than they were to hacking. + +"Illuminati" got its name from a card-game that Steve Jackson himself, +the company's founder and sole owner, had invented. This multi-player +card-game was one of Mr Jackson's best-known, most successful, +most technically innovative products. "Illuminati" was a game +of paranoiac conspiracy in which various antisocial cults warred +covertly to dominate the world. "Illuminati" was hilarious, +and great fun to play, involving flying saucers, the CIA, the KGB, +the phone companies, the Ku Klux Klan, the South American Nazis, +the cocaine cartels, the Boy Scouts, and dozens of other splinter groups +from the twisted depths of Mr. Jackson's professionally fervid imagination. +For the uninitiated, any public discussion of the "Illuminati" card-game +sounded, by turns, utterly menacing or completely insane. + +And then there was SJG's "Car Wars," in which souped-up armored hot-rods +with rocket-launchers and heavy machine-guns did battle on the American +highways of the future. The lively Car Wars discussion on the Illuminati +board featured many meticulous, painstaking discussions of the effects +of grenades, land-mines, flamethrowers and napalm. It sounded like +hacker anarchy files run amuck. + +Mr Jackson and his co-workers earned their daily bread by supplying people +with make-believe adventures and weird ideas. The more far-out, the better. + +Simulation gaming is an unusual pastime, but gamers have not +generally had to beg the permission of the Secret Service to exist. +Wargames and role-playing adventures are an old and honored pastime, +much favored by professional military strategists. Once little-known, +these games are now played by hundreds of thousands of enthusiasts +throughout North America, Europe and Japan. Gaming-books, once restricted +to hobby outlets, now commonly appear in chain-stores like B. Dalton's +and Waldenbooks, and sell vigorously. + +Steve Jackson Games, Inc., of Austin, Texas, was a games company +of the middle rank. In 1989, SJG grossed about a million dollars. +Jackson himself had a good reputation in his industry as a talented +and innovative designer of rather unconventional games, but his company +was something less than a titan of the field--certainly not like the +multimillion-dollar TSR Inc., or Britain's gigantic "Games Workshop." +SJG's Austin headquarters was a modest two-story brick office-suite, +cluttered with phones, photocopiers, fax machines and computers. +It bustled with semi-organized activity and was littered with +glossy promotional brochures and dog-eared science-fiction novels. +Attached to the offices was a large tin-roofed warehouse piled twenty feet +high with cardboard boxes of games and books. Despite the weird imaginings +that went on within it, the SJG headquarters was quite a quotidian, +everyday sort of place. It looked like what it was: a publishers' digs. + +Both "Car Wars" and "Illuminati" were well-known, popular games. +But the mainstay of the Jackson organization was their Generic Universal +Role-Playing System, "G.U.R.P.S." The GURPS system was considered solid +and well-designed, an asset for players. But perhaps the most popular +feature of the GURPS system was that it allowed gaming-masters to design +scenarios that closely resembled well-known books, movies, and other works +of fantasy. Jackson had licensed and adapted works from many science fiction +and fantasy authors. There was GURPS Conan, GURPS Riverworld, +GURPS Horseclans, GURPS Witch World, names eminently familiar +to science-fiction readers. And there was GURPS Special Ops, +from the world of espionage fantasy and unconventional warfare. + +And then there was GURPS Cyberpunk. + +"Cyberpunk" was a term given to certain science fiction writers +who had entered the genre in the 1980s. "Cyberpunk," as the label implies, +had two general distinguishing features. First, its writers had a compelling +interest in information technology, an interest closely akin +to science fiction's earlier fascination with space travel. +And second, these writers were "punks," with all the +distinguishing features that that implies: Bohemian artiness, +youth run wild, an air of deliberate rebellion, funny clothes and hair, +odd politics, a fondness for abrasive rock and roll; in a word, trouble. + +The "cyberpunk" SF writers were a small group of mostly college-educated +white middle-class litterateurs, scattered through the US and Canada. +Only one, Rudy Rucker, a professor of computer science in Silicon Valley, +could rank with even the humblest computer hacker. But, except for +Professor Rucker, the "cyberpunk" authors were not programmers +or hardware experts; they considered themselves artists +(as, indeed, did Professor Rucker). However, these writers +all owned computers, and took an intense and public interest +in the social ramifications of the information industry. + +The cyberpunks had a strong following among the global generation +that had grown up in a world of computers, multinational networks, +and cable television. Their outlook was considered somewhat morbid, +cynical, and dark, but then again, so was the outlook of their +generational peers. As that generation matured and increased +in strength and influence, so did the cyberpunks. +As science-fiction writers went, they were doing +fairly well for themselves. By the late 1980s, +their work had attracted attention from gaming companies, +including Steve Jackson Games, which was planning a cyberpunk +simulation for the flourishing GURPS gaming-system. + +The time seemed ripe for such a product, which had already been proven +in the marketplace. The first games- company out of the gate, +with a product boldly called "Cyberpunk" in defiance of possible +infringement-of-copyright suits, had been an upstart group called +R. Talsorian. Talsorian's Cyberpunk was a fairly decent game, +but the mechanics of the simulation system left a lot to be desired. +Commercially, however, the game did very well. + +The next cyberpunk game had been the even more successful Shadowrun +by FASA Corporation. The mechanics of this game were fine, but the +scenario was rendered moronic by sappy fantasy elements like elves, +trolls, wizards, and dragons--all highly ideologically-incorrect, +according to the hard-edged, high-tech standards of cyberpunk science fiction. + +Other game designers were champing at the bit. Prominent among them +was the Mentor, a gentleman who, like most of his friends in the +Legion of Doom, was quite the cyberpunk devotee. Mentor reasoned +that the time had come for a REAL cyberpunk gaming-book--one that the +princes of computer-mischief in the Legion of Doom could play without +laughing themselves sick. This book, GURPS Cyberpunk, would reek +of culturally on-line authenticity. + +Mentor was particularly well-qualified for this task. +Naturally, he knew far more about computer-intrusion +and digital skullduggery than any previously published +cyberpunk author. Not only that, but he was good at his work. +A vivid imagination, combined with an instinctive feeling +for the working of systems and, especially, the loopholes +within them, are excellent qualities for a professional game designer. + +By March 1st, GURPS Cyberpunk was almost complete, ready to print and ship. +Steve Jackson expected vigorous sales for this item, which, he hoped, +would keep the company financially afloat for several months. +GURPS Cyberpunk, like the other GURPS "modules," was not a "game" +like a Monopoly set, but a BOOK: a bound paperback book the size +of a glossy magazine, with a slick color cover, and pages full of text, +illustrations, tables and footnotes. It was advertised as a game, +and was used as an aid to game-playing, but it was a book, +with an ISBN number, published in Texas, copyrighted, +and sold in bookstores. + +And now, that book, stored on a computer, had gone out the door +in the custody of the Secret Service. + +The day after the raid, Steve Jackson visited the local Secret Service +headquarters with a lawyer in tow. There he confronted Tim Foley +(still in Austin at that time) and demanded his book back. But there +was trouble. GURPS Cyberpunk, alleged a Secret Service agent to astonished +businessman Steve Jackson, was "a manual for computer crime." + +"It's science fiction," Jackson said. + +"No, this is real." + +This statement was repeated several times, by several agents. +Jackson's ominously accurate game had passed from pure, +obscure, small-scale fantasy into the impure, highly publicized, +large-scale fantasy of the Hacker Crackdown. + +No mention was made of the real reason for the search. +According to their search warrant, the raiders had expected +to find the E911 Document stored on Jackson's bulletin board system. +But that warrant was sealed; a procedure that most law enforcement agencies +will use only when lives are demonstrably in danger. The raiders' +true motives were not discovered until the Jackson search-warrant +was unsealed by his lawyers, many months later. The Secret Service, +and the Chicago Computer Fraud and Abuse Task Force, +said absolutely nothing to Steve Jackson about any threat +to the police 911 System. They said nothing about the Atlanta Three, +nothing about Phrack or Knight Lightning, nothing about Terminus. + +Jackson was left to believe that his computers had been seized because +he intended to publish a science fiction book that law enforcement +considered too dangerous to see print. + +This misconception was repeated again and again, for months, +to an ever-widening public audience. It was not the truth of the case; +but as months passed, and this misconception was publicly printed again +and again, it became one of the few publicly known "facts" about +the mysterious Hacker Crackdown. The Secret Service had seized a computer +to stop the publication of a cyberpunk science fiction book. + +The second section of this book, "The Digital Underground," +is almost finished now. We have become acquainted with all +the major figures of this case who actually belong to the +underground milieu of computer intrusion. We have some idea +of their history, their motives, their general modus operandi. +We now know, I hope, who they are, where they came from, +and more or less what they want. In the next section of this book, +"Law and Order," we will leave this milieu and directly enter the +world of America's computer-crime police. + +At this point, however, I have another figure to introduce: myself. + +My name is Bruce Sterling. I live in Austin, Texas, where I am +a science fiction writer by trade: specifically, a CYBERPUNK +science fiction writer. + +Like my "cyberpunk" colleagues in the U.S. and Canada, +I've never been entirely happy with this literary label-- +especially after it became a synonym for computer criminal. +But I did once edit a book of stories by my colleagues, +called Mirrorshades: the Cyberpunk Anthology, and I've +long been a writer of literary-critical cyberpunk manifestos. +I am not a "hacker" of any description, though I do have readers +in the digital underground. + +When the Steve Jackson Games seizure occurred, I naturally took +an intense interest. If "cyberpunk" books were being banned +by federal police in my own home town, I reasonably wondered +whether I myself might be next. Would my computer be seized +by the Secret Service? At the time, I was in possession +of an aging Apple IIe without so much as a hard disk. +If I were to be raided as an author of computer-crime manuals, +the loss of my feeble word-processor would likely provoke more +snickers than sympathy. + +I'd known Steve Jackson for many years. We knew +one another as colleagues, for we frequented +the same local science-fiction conventions. +I'd played Jackson games, and recognized his cleverness; +but he certainly had never struck me as a potential mastermind +of computer crime. + +I also knew a little about computer bulletin-board systems. +In the mid-1980s I had taken an active role in an Austin board +called "SMOF-BBS," one of the first boards dedicated to science fiction. +I had a modem, and on occasion I'd logged on to Illuminati, +which always looked entertainly wacky, but certainly harmless enough. + +At the time of the Jackson seizure, I had no experience +whatsoever with underground boards. But I knew that no one +on Illuminati talked about breaking into systems illegally, +or about robbing phone companies. Illuminati didn't even +offer pirated computer games. Steve Jackson, like many creative artists, +was markedly touchy about theft of intellectual property. + +It seemed to me that Jackson was either seriously suspected +of some crime--in which case, he would be charged soon, +and would have his day in court--or else he was innocent, +in which case the Secret Service would quickly return his equipment, +and everyone would have a good laugh. I rather expected the good laugh. +The situation was not without its comic side. The raid, known +as the "Cyberpunk Bust" in the science fiction community, +was winning a great deal of free national publicity both +for Jackson himself and the "cyberpunk" science fiction +writers generally. + +Besides, science fiction people are used to being misinterpreted. +Science fiction is a colorful, disreputable, slipshod occupation, +full of unlikely oddballs, which, of course, is why we like it. +Weirdness can be an occupational hazard in our field. People who +wear Halloween costumes are sometimes mistaken for monsters. + +Once upon a time--back in 1939, in New York City-- +science fiction and the U.S. Secret Service collided in +a comic case of mistaken identity. This weird incident +involved a literary group quite famous in science fiction, +known as "the Futurians," whose membership included +such future genre greats as Isaac Asimov, Frederik Pohl, +and Damon Knight. The Futurians were every bit as +offbeat and wacky as any of their spiritual descendants, +including the cyberpunks, and were given to communal living, +spontaneous group renditions of light opera, and midnight fencing +exhibitions on the lawn. The Futurians didn't have bulletin +board systems, but they did have the technological equivalent +in 1939--mimeographs and a private printing press. These were +in steady use, producing a stream of science-fiction fan magazines, +literary manifestos, and weird articles, which were picked up +in ink-sticky bundles by a succession of strange, gangly, +spotty young men in fedoras and overcoats. + +The neighbors grew alarmed at the antics of the Futurians +and reported them to the Secret Service as suspected counterfeiters. +In the winter of 1939, a squad of USSS agents with drawn guns burst into +"Futurian House," prepared to confiscate the forged currency and illicit +printing presses. There they discovered a slumbering science fiction fan +named George Hahn, a guest of the Futurian commune who had just arrived +in New York. George Hahn managed to explain himself and his group, +and the Secret Service agents left the Futurians in peace henceforth. +(Alas, Hahn died in 1991, just before I had discovered this astonishing +historical parallel, and just before I could interview him for this book.) + +But the Jackson case did not come to a swift and comic end. +No quick answers came his way, or mine; no swift reassurances +that all was right in the digital world, that matters were well +in hand after all. Quite the opposite. In my alternate role +as a sometime pop-science journalist, I interviewed Jackson +and his staff for an article in a British magazine. +The strange details of the raid left me more concerned than ever. +Without its computers, the company had been financially +and operationally crippled. Half the SJG workforce, +a group of entirely innocent people, had been sorrowfully fired, +deprived of their livelihoods by the seizure. It began to dawn on me +that authors--American writers--might well have their computers seized, +under sealed warrants, without any criminal charge; and that, +as Steve Jackson had discovered, there was no immediate recourse for this. +This was no joke; this wasn't science fiction; this was real. + +I determined to put science fiction aside until I had discovered +what had happened and where this trouble had come from. +It was time to enter the purportedly real world of electronic +free expression and computer crime. Hence, this book. +Hence, the world of the telcos; and the world of the digital underground; +and next, the world of the police. + + + +PART THREE: LAW AND ORDER + + +Of the various anti-hacker activities of 1990, "Operation Sundevil" +had by far the highest public profile. The sweeping, nationwide +computer seizures of May 8, 1990 were unprecedented in scope and highly, +if rather selectively, publicized. + +Unlike the efforts of the Chicago Computer Fraud and Abuse Task Force, +"Operation Sundevil" was not intended to combat "hacking" in the sense +of computer intrusion or sophisticated raids on telco switching stations. +Nor did it have anything to do with hacker misdeeds with AT&T's software, +or with Southern Bell's proprietary documents. + +Instead, "Operation Sundevil" was a crackdown on those traditional scourges +of the digital underground: credit-card theft and telephone code abuse. +The ambitious activities out of Chicago, and the somewhat lesser-known +but vigorous anti-hacker actions of the New York State Police in 1990, +were never a part of "Operation Sundevil" per se, which was based in Arizona. + +Nevertheless, after the spectacular May 8 raids, the public, misled by +police secrecy, hacker panic, and a puzzled national press-corps, +conflated all aspects of the nationwide crackdown in 1990 under +the blanket term "Operation Sundevil." "Sundevil" is still the best-known +synonym for the crackdown of 1990. But the Arizona organizers of "Sundevil" +did not really deserve this reputation--any more, for instance, than all +hackers deserve a reputation as "hackers." + +There was some justice in this confused perception, though. +For one thing, the confusion was abetted by the Washington office +of the Secret Service, who responded to Freedom of Information Act +requests on "Operation Sundevil" by referring investigators +to the publicly known cases of Knight Lightning and the Atlanta Three. +And "Sundevil" was certainly the largest aspect of the Crackdown, +the most deliberate and the best-organized. As a crackdown on electronic +fraud, "Sundevil" lacked the frantic pace of the war on the Legion of Doom; +on the contrary, Sundevil's targets were picked out with cool deliberation +over an elaborate investigation lasting two full years. + +And once again the targets were bulletin board systems. + +Boards can be powerful aids to organized fraud. Underground boards carry +lively, extensive, detailed, and often quite flagrant "discussions" of +lawbreaking techniques and lawbreaking activities. "Discussing" crime +in the abstract, or "discussing" the particulars of criminal cases, +is not illegal--but there are stern state and federal laws against +coldbloodedly conspiring in groups in order to commit crimes. + +In the eyes of police, people who actively conspire to break the law +are not regarded as "clubs," "debating salons," "users' groups," or +"free speech advocates." Rather, such people tend to find themselves +formally indicted by prosecutors as "gangs," "racketeers," "corrupt +organizations" and "organized crime figures." + +What's more, the illicit data contained on outlaw boards goes well beyond +mere acts of speech and/or possible criminal conspiracy. As we have seen, +it was common practice in the digital underground to post purloined telephone +codes on boards, for any phreak or hacker who cared to abuse them. Is posting +digital booty of this sort supposed to be protected by the First Amendment? +Hardly--though the issue, like most issues in cyberspace, is not entirely +resolved. Some theorists argue that to merely RECITE a number publicly +is not illegal--only its USE is illegal. But anti-hacker police point out +that magazines and newspapers (more traditional forms of free expression) +never publish stolen telephone codes (even though this might well +raise their circulation). + +Stolen credit card numbers, being riskier and more valuable, +were less often publicly posted on boards--but there is no question +that some underground boards carried "carding" traffic, +generally exchanged through private mail. + +Underground boards also carried handy programs for "scanning" telephone +codes and raiding credit card companies, as well as the usual obnoxious +galaxy of pirated software, cracked passwords, blue-box schematics, +intrusion manuals, anarchy files, porn files, and so forth. + +But besides their nuisance potential for the spread of illicit knowledge, +bulletin boards have another vitally interesting aspect for the +professional investigator. Bulletin boards are cram-full of EVIDENCE. +All that busy trading of electronic mail, all those hacker boasts, +brags and struts, even the stolen codes and cards, can be neat, +electronic, real-time recordings of criminal activity. +As an investigator, when you seize a pirate board, you have +scored a coup as effective as tapping phones or intercepting mail. +However, you have not actually tapped a phone or intercepted a letter. +The rules of evidence regarding phone-taps and mail interceptions are old, +stern and well-understood by police, prosecutors and defense attorneys alike. +The rules of evidence regarding boards are new, waffling, and understood +by nobody at all. + +Sundevil was the largest crackdown on boards in world history. +On May 7, 8, and 9, 1990, about forty-two computer systems were seized. +Of those forty-two computers, about twenty-five actually were running boards. +(The vagueness of this estimate is attributable to the vagueness of +(a) what a "computer system" is, and (b) what it actually means to +"run a board" with one--or with two computers, or with three.) + +About twenty-five boards vanished into police custody in May 1990. +As we have seen, there are an estimated 30,000 boards in America today. +If we assume that one board in a hundred is up to no good with codes +and cards (which rather flatters the honesty of the board-using community), +then that would leave 2,975 outlaw boards untouched by Sundevil. +Sundevil seized about one tenth of one percent of all computer +bulletin boards in America. Seen objectively, this is something less +than a comprehensive assault. In 1990, Sundevil's organizers-- +the team at the Phoenix Secret Service office, and the Arizona +Attorney General's office-- had a list of at least THREE HUNDRED +boards that they considered fully deserving of search and seizure warrants. +The twenty-five boards actually seized were merely among the most obvious +and egregious of this much larger list of candidates. All these boards +had been examined beforehand--either by informants, who had passed printouts +to the Secret Service, or by Secret Service agents themselves, who not only +come equipped with modems but know how to use them. + +There were a number of motives for Sundevil. First, it offered +a chance to get ahead of the curve on wire-fraud crimes. +Tracking back credit-card ripoffs to their perpetrators +can be appallingly difficult. If these miscreants +have any kind of electronic sophistication, they can snarl +their tracks through the phone network into a mind-boggling, +untraceable mess, while still managing to "reach out and rob someone." +Boards, however, full of brags and boasts, codes and cards, +offer evidence in the handy congealed form. + +Seizures themselves--the mere physical removal of machines-- +tends to take the pressure off. During Sundevil, a large number +of code kids, warez d00dz, and credit card thieves would be deprived +of those boards--their means of community and conspiracy--in one swift blow. +As for the sysops themselves (commonly among the boldest offenders) +they would be directly stripped of their computer equipment, +and rendered digitally mute and blind. + +And this aspect of Sundevil was carried out with great success. +Sundevil seems to have been a complete tactical surprise-- +unlike the fragmentary and continuing seizures of the war on the +Legion of Doom, Sundevil was precisely timed and utterly overwhelming. +At least forty "computers" were seized during May 7, 8 and 9, 1990, +in Cincinnati, Detroit, Los Angeles, Miami, Newark, Phoenix, Tucson, +Richmond, San Diego, San Jose, Pittsburgh and San Francisco. +Some cities saw multiple raids, such as the five separate raids +in the New York City environs. Plano, Texas (essentially a suburb of +the Dallas/Fort Worth metroplex, and a hub of the telecommunications industry) +saw four computer seizures. Chicago, ever in the forefront, saw its own +local Sundevil raid, briskly carried out by Secret Service agents +Timothy Foley and Barbara Golden. + +Many of these raids occurred, not in the cities proper, +but in associated white-middle class suburbs--places like +Mount Lebanon, Pennsylvania and Clark Lake, Michigan. +There were a few raids on offices; most took place in people's homes, +the classic hacker basements and bedrooms. + +The Sundevil raids were searches and seizures, not a group of mass arrests. +There were only four arrests during Sundevil. "Tony the Trashman," +a longtime teenage bete noire of the Arizona Racketeering unit, +was arrested in Tucson on May 9. "Dr. Ripco," sysop of an outlaw board +with the misfortune to exist in Chicago itself, was also arrested-- +on illegal weapons charges. Local units also arrested a 19-year-old +female phone phreak named "Electra" in Pennsylvania, and a male juvenile +in California. Federal agents however were not seeking arrests, but computers. + +Hackers are generally not indicted (if at all) until the evidence +in their seized computers is evaluated--a process that can take weeks, +months--even years. When hackers are arrested on the spot, it's generally +an arrest for other reasons. Drugs and/or illegal weapons show up in a good +third of anti-hacker computer seizures (though not during Sundevil). + +That scofflaw teenage hackers (or their parents) should have marijuana +in their homes is probably not a shocking revelation, but the surprisingly +common presence of illegal firearms in hacker dens is a bit disquieting. +A Personal Computer can be a great equalizer for the techno-cowboy-- +much like that more traditional American "Great Equalizer," +the Personal Sixgun. Maybe it's not all that surprising +that some guy obsessed with power through illicit technology +would also have a few illicit high-velocity-impact devices around. +An element of the digital underground particularly dotes on those +"anarchy philes," and this element tends to shade into the crackpot milieu +of survivalists, gun-nuts, anarcho-leftists and the ultra-libertarian +right-wing. + +This is not to say that hacker raids to date have uncovered any +major crack-dens or illegal arsenals; but Secret Service agents +do not regard "hackers" as "just kids." They regard hackers as +unpredictable people, bright and slippery. It doesn't help matters +that the hacker himself has been "hiding behind his keyboard" +all this time. Commonly, police have no idea what he looks like. +This makes him an unknown quantity, someone best treated with +proper caution. + +To date, no hacker has come out shooting, though they do sometimes brag on +boards that they will do just that. Threats of this sort are taken seriously. +Secret Service hacker raids tend to be swift, comprehensive, well-manned +(even over-manned); and agents generally burst through every door +in the home at once, sometimes with drawn guns. Any potential resistance +is swiftly quelled. Hacker raids are usually raids on people's homes. +It can be a very dangerous business to raid an American home; +people can panic when strangers invade their sanctum. Statistically speaking, +the most dangerous thing a policeman can do is to enter someone's home. +(The second most dangerous thing is to stop a car in traffic.) +People have guns in their homes. More cops are hurt in homes +than are ever hurt in biker bars or massage parlors. + +But in any case, no one was hurt during Sundevil, +or indeed during any part of the Hacker Crackdown. + +Nor were there any allegations of any physical mistreatment of a suspect. +Guns were pointed, interrogations were sharp and prolonged; but no one +in 1990 claimed any act of brutality by any crackdown raider. + +In addition to the forty or so computers, Sundevil reaped floppy disks +in particularly great abundance--an estimated 23,000 of them, which +naturally included every manner of illegitimate data: pirated games, +stolen codes, hot credit card numbers, the complete text and software +of entire pirate bulletin-boards. These floppy disks, which remain +in police custody today, offer a gigantic, almost embarrassingly +rich source of possible criminal indictments. These 23,000 floppy disks +also include a thus-far unknown quantity of legitimate computer games, +legitimate software, purportedly "private" mail from boards, +business records, and personal correspondence of all kinds. + +Standard computer-crime search warrants lay great emphasis on seizing +written documents as well as computers--specifically including photocopies, +computer printouts, telephone bills, address books, logs, notes, +memoranda and correspondence. In practice, this has meant that diaries, +gaming magazines, software documentation, nonfiction books on hacking +and computer security, sometimes even science fiction novels, have all +vanished out the door in police custody. A wide variety of electronic items +have been known to vanish as well, including telephones, televisions, answering +machines, Sony Walkmans, desktop printers, compact disks, and audiotapes. + +No fewer than 150 members of the Secret Service were sent into +the field during Sundevil. They were commonly accompanied by +squads of local and/or state police. Most of these officers-- +especially the locals--had never been on an anti-hacker raid before. +(This was one good reason, in fact, why so many of them were invited along +in the first place.) Also, the presence of a uniformed police officer +assures the raidees that the people entering their homes are, in fact, police. +Secret Service agents wear plain clothes. So do the telco security experts +who commonly accompany the Secret Service on raids (and who make no particular +effort to identify themselves as mere employees of telephone companies). + +A typical hacker raid goes something like this. First, police storm in +rapidly, through every entrance, with overwhelming force, +in the assumption that this tactic will keep casualties to a minimum. +Second, possible suspects are immediately removed from the vicinity +of any and all computer systems, so that they will have no chance +to purge or destroy computer evidence. Suspects are herded into a room +without computers, commonly the living room, and kept under guard-- +not ARMED guard, for the guns are swiftly holstered, but under guard +nevertheless. They are presented with the search warrant and warned +that anything they say may be held against them. Commonly they have +a great deal to say, especially if they are unsuspecting parents. + +Somewhere in the house is the "hot spot"--a computer tied to a phone +line (possibly several computers and several phones). Commonly it's +a teenager's bedroom, but it can be anywhere in the house; +there may be several such rooms. This "hot spot" is put in charge +of a two-agent team, the "finder" and the "recorder." The "finder" +is computer-trained, commonly the case agent who has actually obtained +the search warrant from a judge. He or she understands what is being sought, +and actually carries out the seizures: unplugs machines, opens drawers, +desks, files, floppy-disk containers, etc. The "recorder" photographs +all the equipment, just as it stands--especially the tangle of +wired connections in the back, which can otherwise be a real nightmare +to restore. The recorder will also commonly photograph every room +in the house, lest some wily criminal claim that the police had robbed him +during the search. Some recorders carry videocams or tape recorders; +however, it's more common for the recorder to simply take written notes. +Objects are described and numbered as the finder seizes them, generally +on standard preprinted police inventory forms. + +Even Secret Service agents were not, and are not, expert computer users. +They have not made, and do not make, judgements on the fly about potential +threats posed by various forms of equipment. They may exercise discretion; +they may leave Dad his computer, for instance, but they don't HAVE to. +Standard computer-crime search warrants, which date back to the early 80s, +use a sweeping language that targets computers, most anything attached +to a computer, most anything used to operate a computer--most anything +that remotely resembles a computer--plus most any and all written documents +surrounding it. Computer-crime investigators have strongly urged agents +to seize the works. + +In this sense, Operation Sundevil appears to have been a complete success. +Boards went down all over America, and were shipped en masse to the computer +investigation lab of the Secret Service, in Washington DC, along with the +23,000 floppy disks and unknown quantities of printed material. + +But the seizure of twenty-five boards, and the multi-megabyte mountains +of possibly useful evidence contained in these boards (and in their owners' +other computers, also out the door), were far from the only motives for +Operation Sundevil. An unprecedented action of great ambition and size, +Sundevil's motives can only be described as political. It was a +public-relations effort, meant to pass certain messages, meant to make +certain situations clear: both in the mind of the general public, +and in the minds of various constituencies of the electronic community. + + First --and this motivation was vital--a "message" would be sent from +law enforcement to the digital underground. This very message was recited +in so many words by Garry M. Jenkins, the Assistant Director of the +US Secret Service, at the Sundevil press conference in Phoenix on +May 9, 1990, immediately after the raids. In brief, hackers were +mistaken in their foolish belief that they could hide behind the +"relative anonymity of their computer terminals." On the contrary, +they should fully understand that state and federal cops were +actively patrolling the beat in cyberspace--that they were +on the watch everywhere, even in those sleazy and secretive +dens of cybernetic vice, the underground boards. + +This is not an unusual message for police to publicly convey to crooks. +The message is a standard message; only the context is new. + +In this respect, the Sundevil raids were the digital equivalent +of the standard vice-squad crackdown on massage parlors, porno bookstores, +head-shops, or floating crap-games. There may be few or no arrests in a raid +of this sort; no convictions, no trials, no interrogations. In cases of this +sort, police may well walk out the door with many pounds of sleazy magazines, +X-rated videotapes, sex toys, gambling equipment, baggies of marijuana. . . . + +Of course, if something truly horrendous is discovered by the raiders, +there will be arrests and prosecutions. Far more likely, however, +there will simply be a brief but sharp disruption of the closed +and secretive world of the nogoodniks. There will be "street hassle." +"Heat." "Deterrence." And, of course, the immediate loss of the seized goods. +It is very unlikely that any of this seized material will ever be returned. +Whether charged or not, whether convicted or not, the perpetrators will +almost surely lack the nerve ever to ask for this stuff to be given back. + +Arrests and trials--putting people in jail--may involve all kinds of +formal legalities; but dealing with the justice system is far from the only +task of police. Police do not simply arrest people. They don't simply +put people in jail. That is not how the police perceive their jobs. +Police "protect and serve." Police "keep the peace," they "keep public order." +Like other forms of public relations, keeping public order is not an +exact science. Keeping public order is something of an art-form. + +If a group of tough-looking teenage hoodlums was loitering on a street-corner, +no one would be surprised to see a street-cop arrive and sternly order +them to "break it up." On the contrary, the surprise would come if one +of these ne'er-do-wells stepped briskly into a phone-booth, +called a civil rights lawyer, and instituted a civil suit +in defense of his Constitutional rights of free speech +and free assembly. But something much along this line +was one of the many anomolous outcomes of the Hacker Crackdown. + +Sundevil also carried useful "messages" for other constituents of +the electronic community. These messages may not have been read +aloud from the Phoenix podium in front of the press corps, +but there was little mistaking their meaning. There was a message +of reassurance for the primary victims of coding and carding: +the telcos, and the credit companies. Sundevil was greeted with joy +by the security officers of the electronic business community. +After years of high-tech harassment and spiralling revenue losses, +their complaints of rampant outlawry were being taken seriously by +law enforcement. No more head-scratching or dismissive shrugs; +no more feeble excuses about "lack of computer-trained officers" or +the low priority of "victimless" white-collar telecommunication crimes. + +Computer-crime experts have long believed that computer-related offenses +are drastically under-reported. They regard this as a major open scandal +of their field. Some victims are reluctant to come forth, because they +believe that police and prosecutors are not computer-literate, +and can and will do nothing. Others are embarrassed by +their vulnerabilities, and will take strong measures +to avoid any publicity; this is especially true of banks, +who fear a loss of investor confidence should an embezzlement-case +or wire-fraud surface. And some victims are so helplessly confused +by their own high technology that they never even realize that +a crime has occurred--even when they have been fleeced to the bone. + +The results of this situation can be dire. +Criminals escape apprehension and punishment. +The computer-crime units that do exist, can't get work. +The true scope of computer-crime: its size, its real nature, +the scope of its threats, and the legal remedies for it-- +all remain obscured. + +Another problem is very little publicized, but it is a cause +of genuine concern. Where there is persistent crime, +but no effective police protection, then vigilantism can result. +Telcos, banks, credit companies, the major corporations who +maintain extensive computer networks vulnerable to hacking +--these organizations are powerful, wealthy, and +politically influential. They are disinclined to be +pushed around by crooks (or by most anyone else, +for that matter). They often maintain well-organized +private security forces, commonly run by +experienced veterans of military and police units, +who have left public service for the greener pastures +of the private sector. For police, the corporate +security manager can be a powerful ally; but if this +gentleman finds no allies in the police, and the +pressure is on from his board-of-directors, +he may quietly take certain matters into his own hands. + +Nor is there any lack of disposable hired-help in the +corporate security business. Private security agencies-- +the `security business' generally--grew explosively in the 1980s. +Today there are spooky gumshoed armies of "security consultants," +"rent-a- cops," "private eyes," "outside experts"--every manner +of shady operator who retails in "results" and discretion. +Or course, many of these gentlemen and ladies may be paragons +of professional and moral rectitude. But as anyone +who has read a hard-boiled detective novel knows, +police tend to be less than fond of this sort +of private-sector competition. + +Companies in search of computer-security have even been +known to hire hackers. Police shudder at this prospect. + +Police treasure good relations with the business community. +Rarely will you see a policeman so indiscreet as to allege +publicly that some major employer in his state or city has succumbed +to paranoia and gone off the rails. Nevertheless, +police --and computer police in particular--are aware +of this possibility. Computer-crime police can and do +spend up to half of their business hours just doing +public relations: seminars, "dog and pony shows," +sometimes with parents' groups or computer users, +but generally with their core audience: the likely +victims of hacking crimes. These, of course, are telcos, +credit card companies and large computer-equipped corporations. +The police strongly urge these people, as good citizens, +to report offenses and press criminal charges; +they pass the message that there is someone in authority who cares, +understands, and, best of all, will take useful action +should a computer-crime occur. + +But reassuring talk is cheap. Sundevil offered action. + +The final message of Sundevil was intended for internal consumption +by law enforcement. Sundevil was offered as proof that the community +of American computer-crime police had come of age. Sundevil was +proof that enormous things like Sundevil itself could now be accomplished. +Sundevil was proof that the Secret Service and its local law-enforcement +allies could act like a well-oiled machine--(despite the hampering use +of those scrambled phones). It was also proof that the Arizona Organized +Crime and Racketeering Unit--the sparkplug of Sundevil--ranked with the best +in the world in ambition, organization, and sheer conceptual daring. + +And, as a final fillip, Sundevil was a message from the Secret Service +to their longtime rivals in the Federal Bureau of Investigation. +By Congressional fiat, both USSS and FBI formally share jurisdiction +over federal computer-crimebusting activities. Neither of these groups +has ever been remotely happy with this muddled situation. It seems to +suggest that Congress cannot make up its mind as to which of these groups +is better qualified. And there is scarcely a G-man or a Special Agent +anywhere without a very firm opinion on that topic. + +# + +For the neophyte, one of the most puzzling aspects of the crackdown +on hackers is why the United States Secret Service has anything at all +to do with this matter. + +The Secret Service is best known for its primary public role: +its agents protect the President of the United States. +They also guard the President's family, the Vice President and his family, +former Presidents, and Presidential candidates. They sometimes guard +foreign dignitaries who are visiting the United States, especially foreign +heads of state, and have been known to accompany American officials +on diplomatic missions overseas. + +Special Agents of the Secret Service don't wear uniforms, but the +Secret Service also has two uniformed police agencies. There's the +former White House Police (now known as the Secret Service Uniformed Division, +since they currently guard foreign embassies in Washington, as well as the +White House itself). And there's the uniformed Treasury Police Force. + +The Secret Service has been charged by Congress with a number +of little-known duties. They guard the precious metals in Treasury vaults. +They guard the most valuable historical documents of the United States: +originals of the Constitution, the Declaration of Independence, +Lincoln's Second Inaugural Address, an American-owned copy of +the Magna Carta, and so forth. Once they were assigned to guard +the Mona Lisa, on her American tour in the 1960s. + +The entire Secret Service is a division of the Treasury Department. +Secret Service Special Agents (there are about 1,900 of them) +are bodyguards for the President et al, but they all work for the Treasury. +And the Treasury (through its divisions of the U.S. Mint and the +Bureau of Engraving and Printing) prints the nation's money. + +As Treasury police, the Secret Service guards the nation's currency; +it is the only federal law enforcement agency with direct jurisdiction +over counterfeiting and forgery. It analyzes documents for authenticity, +and its fight against fake cash is still quite lively (especially since +the skilled counterfeiters of Medellin, Columbia have gotten into the act). +Government checks, bonds, and other obligations, which exist in untold +millions and are worth untold billions, are common targets for forgery, +which the Secret Service also battles. It even handles forgery +of postage stamps. + +But cash is fading in importance today as money has become electronic. +As necessity beckoned, the Secret Service moved from fighting the +counterfeiting of paper currency and the forging of checks, +to the protection of funds transferred by wire. + +From wire-fraud, it was a simple skip-and-jump to what is formally +known as "access device fraud." Congress granted the Secret Service +the authority to investigate "access device fraud" under Title 18 +of the United States Code (U.S.C. Section 1029). + +The term "access device" seems intuitively simple. It's some kind +of high-tech gizmo you use to get money with. It makes good sense +to put this sort of thing in the charge of counterfeiting and +wire-fraud experts. + +However, in Section 1029, the term "access device" is very +generously defined. An access device is: "any card, plate, +code, account number, or other means of account access +that can be used, alone or in conjunction with another access device, +to obtain money, goods, services, or any other thing of value, +or that can be used to initiate a transfer of funds." + +"Access device" can therefore be construed to include credit cards +themselves (a popular forgery item nowadays). It also includes credit card +account NUMBERS, those standards of the digital underground. The same goes +for telephone charge cards (an increasingly popular item with telcos, +who are tired of being robbed of pocket change by phone-booth thieves). +And also telephone access CODES, those OTHER standards of the digital +underground. (Stolen telephone codes may not "obtain money," but they +certainly do obtain valuable "services," which is specifically forbidden +by Section 1029.) + +We can now see that Section 1029 already pits the United States Secret Service +directly against the digital underground, without any mention at all of +the word "computer." + +Standard phreaking devices, like "blue boxes," used to steal phone service +from old-fashioned mechanical switches, are unquestionably "counterfeit +access devices." Thanks to Sec.1029, it is not only illegal to USE +counterfeit access devices, but it is even illegal to BUILD them. +"Producing," "designing" "duplicating" or "assembling" blue boxes +are all federal crimes today, and if you do this, the Secret Service +has been charged by Congress to come after you. + +Automatic Teller Machines, which replicated all over America during the 1980s, +are definitely "access devices," too, and an attempt to tamper with their +punch-in codes and plastic bank cards falls directly under Sec. 1029. + +Section 1029 is remarkably elastic. Suppose you find a computer password +in somebody's trash. That password might be a "code"--it's certainly a +"means of account access." Now suppose you log on to a computer +and copy some software for yourself. You've certainly obtained +"service" (computer service) and a "thing of value" (the software). +Suppose you tell a dozen friends about your swiped password, +and let them use it, too. Now you're "trafficking in unauthorized +access devices." And when the Prophet, a member of the Legion of Doom, +passed a stolen telephone company document to Knight Lightning +at Phrack magazine, they were both charged under Sec. 1029! + +There are two limitations on Section 1029. First, the offense must +"affect interstate or foreign commerce" in order to become a matter +of federal jurisdiction. The term "affecting commerce" is not well defined; +but you may take it as a given that the Secret Service can take an interest +if you've done most anything that happens to cross a state line. +State and local police can be touchy about their jurisdictions, +and can sometimes be mulish when the feds show up. But when it comes +to computer-crime, the local police are pathetically grateful +for federal help--in fact they complain that they can't get enough of it. +If you're stealing long-distance service, you're almost certainly crossing +state lines, and you're definitely "affecting the interstate commerce" +of the telcos. And if you're abusing credit cards by ordering stuff +out of glossy catalogs from, say, Vermont, you're in for it. + +The second limitation is money. As a rule, the feds don't pursue +penny-ante offenders. Federal judges will dismiss cases that appear +to waste their time. Federal crimes must be serious; Section 1029 +specifies a minimum loss of a thousand dollars. + +We now come to the very next section of Title 18, which is Section 1030, +"Fraud and related activity in connection with computers." This statute +gives the Secret Service direct jurisdiction over acts of computer intrusion. +On the face of it, the Secret Service would now seem to command the field. +Section 1030, however, is nowhere near so ductile as Section 1029. + +The first annoyance is Section 1030(d), which reads: + +"(d) The United States Secret Service shall, +IN ADDITION TO ANY OTHER AGENCY HAVING SUCH AUTHORITY, +have the authority to investigate offenses under this section. +Such authority of the United States Secret Service shall be +exercised in accordance with an agreement which shall be entered +into by the Secretary of the Treasury AND THE ATTORNEY GENERAL." +(Author's italics.) [Represented by capitals.] + +The Secretary of the Treasury is the titular head of the Secret Service, +while the Attorney General is in charge of the FBI. In Section (d), +Congress shrugged off responsibility for the computer-crime turf-battle +between the Service and the Bureau, and made them fight it out all +by themselves. The result was a rather dire one for the Secret Service, +for the FBI ended up with exclusive jurisdiction over computer break-ins +having to do with national security, foreign espionage, federally insured +banks, and U.S. military bases, while retaining joint jurisdiction over +all the other computer intrusions. Essentially, when it comes to Section 1030, +the FBI not only gets the real glamor stuff for itself, but can peer over the +shoulder of the Secret Service and barge in to meddle whenever it suits them. + +The second problem has to do with the dicey term +"Federal interest computer." Section 1030(a)(2) +makes it illegal to "access a computer without authorization" +if that computer belongs to a financial institution or an issuer +of credit cards (fraud cases, in other words). Congress was quite +willing to give the Secret Service jurisdiction over +money-transferring computers, but Congress balked at +letting them investigate any and all computer intrusions. +Instead, the USSS had to settle for the money machines +and the "Federal interest computers." A "Federal interest computer" +is a computer which the government itself owns, or is using. +Large networks of interstate computers, linked over state lines, +are also considered to be of "Federal interest." (This notion of +"Federal interest" is legally rather foggy and has never been +clearly defined in the courts. The Secret Service has never yet +had its hand slapped for investigating computer break-ins that were NOT +of "Federal interest," but conceivably someday this might happen.) + +So the Secret Service's authority over "unauthorized access" +to computers covers a lot of territory, but by no means the +whole ball of cyberspatial wax. If you are, for instance, +a LOCAL computer retailer, or the owner of a LOCAL bulletin +board system, then a malicious LOCAL intruder can break in, +crash your system, trash your files and scatter viruses, +and the U.S. Secret Service cannot do a single thing about it. + +At least, it can't do anything DIRECTLY. But the Secret Service +will do plenty to help the local people who can. + +The FBI may have dealt itself an ace off the bottom of the deck +when it comes to Section 1030; but that's not the whole story; +that's not the street. What's Congress thinks is one thing, +and Congress has been known to change its mind. The REAL +turf-struggle is out there in the streets where it's happening. +If you're a local street-cop with a computer problem, +the Secret Service wants you to know where you can find +the real expertise. While the Bureau crowd are off having +their favorite shoes polished--(wing-tips)--and making derisive +fun of the Service's favorite shoes--("pansy-ass tassels")-- +the tassel-toting Secret Service has a crew of ready-and-able +hacker-trackers installed in the capital of every state in the Union. +Need advice? They'll give you advice, or at least point you in +the right direction. Need training? They can see to that, too. + +If you're a local cop and you call in the FBI, the FBI +(as is widely and slanderously rumored) will order you around +like a coolie, take all the credit for your busts, +and mop up every possible scrap of reflected glory. +The Secret Service, on the other hand, doesn't brag a lot. +They're the quiet types. VERY quiet. Very cool. Efficient. +High-tech. Mirrorshades, icy stares, radio ear-plugs, +an Uzi machine-pistol tucked somewhere in that well-cut jacket. +American samurai, sworn to give their lives to protect our President. +"The granite agents." Trained in martial arts, absolutely fearless. +Every single one of 'em has a top-secret security clearance. +Something goes a little wrong, you're not gonna hear any whining +and moaning and political buck-passing out of these guys. + +The facade of the granite agent is not, of course, the reality. +Secret Service agents are human beings. And the real glory +in Service work is not in battling computer crime--not yet, +anyway--but in protecting the President. The real glamour +of Secret Service work is in the White House Detail. +If you're at the President's side, then the kids and the wife +see you on television; you rub shoulders with the most powerful +people in the world. That's the real heart of Service work, +the number one priority. More than one computer investigation +has stopped dead in the water when Service agents vanished at +the President's need. + +There's romance in the work of the Service. The intimate access +to circles of great power; the esprit-de-corps of a highly trained +and disciplined elite; the high responsibility of defending the +Chief Executive; the fulfillment of a patriotic duty. And as police +work goes, the pay's not bad. But there's squalor in Service work, too. +You may get spat upon by protesters howling abuse--and if they get violent, +if they get too close, sometimes you have to knock one of them down-- +discreetly. + +The real squalor in Service work is drudgery such as "the quarterlies," +traipsing out four times a year, year in, year out, to interview the various +pathetic wretches, many of them in prisons and asylums, who have seen fit +to threaten the President's life. And then there's the grinding stress +of searching all those faces in the endless bustling crowds, looking for +hatred, looking for psychosis, looking for the tight, nervous face +of an Arthur Bremer, a Squeaky Fromme, a Lee Harvey Oswald. +It's watching all those grasping, waving hands for sudden movements, +while your ears strain at your radio headphone for the long-rehearsed +cry of "Gun!" + +It's poring, in grinding detail, over the biographies of every rotten +loser who ever shot at a President. It's the unsung work of the +Protective Research Section, who study scrawled, anonymous death threats +with all the meticulous tools of anti-forgery techniques. + +And it's maintaining the hefty computerized files on anyone +who ever threatened the President's life. Civil libertarians +have become increasingly concerned at the Government's use +of computer files to track American citizens--but the +Secret Service file of potential Presidential assassins, +which has upward of twenty thousand names, rarely causes +a peep of protest. If you EVER state that you intend to +kill the President, the Secret Service will want to know +and record who you are, where you are, what you are, +and what you're up to. If you're a serious threat-- +if you're officially considered "of protective interest"-- +then the Secret Service may well keep tabs on you +for the rest of your natural life. + +Protecting the President has first call on all the Service's resources. +But there's a lot more to the Service's traditions and history than +standing guard outside the Oval Office. + +The Secret Service is the nation's oldest general federal +law-enforcement agency. Compared to the Secret Service, +the FBI are new-hires and the CIA are temps. The Secret Service +was founded 'way back in 1865, at the suggestion of Hugh McCulloch, +Abraham Lincoln's Secretary of the Treasury. McCulloch wanted +a specialized Treasury police to combat counterfeiting. +Abraham Lincoln agreed that this seemed a good idea, and, +with a terrible irony, Abraham Lincoln was shot that +very night by John Wilkes Booth. + +The Secret Service originally had nothing to do with protecting Presidents. +They didn't take this on as a regular assignment until after the Garfield +assassination in 1881. And they didn't get any Congressional money for it +until President McKinley was shot in 1901. The Service was originally +designed for one purpose: destroying counterfeiters. + +# + +There are interesting parallels between the Service's +nineteenth-century entry into counterfeiting, +and America's twentieth-century entry into computer-crime. + +In 1865, America's paper currency was a terrible muddle. +Security was drastically bad. Currency was printed on the spot +by local banks in literally hundreds of different designs. +No one really knew what the heck a dollar bill was supposed to look like. +Bogus bills passed easily. If some joker told you that a one-dollar bill +from the Railroad Bank of Lowell, Massachusetts had a woman leaning on +a shield, with a locomotive, a cornucopia, a compass, various agricultural +implements, a railroad bridge, and some factories, then you pretty much had +to take his word for it. (And in fact he was telling the truth!) + +SIXTEEN HUNDRED local American banks designed and printed their own +paper currency, and there were no general standards for security. +Like a badly guarded node in a computer network, badly designed bills +were easy to fake, and posed a security hazard for the entire monetary system. + +No one knew the exact extent of the threat to the currency. +There were panicked estimates that as much as a third of +the entire national currency was faked. Counterfeiters-- +known as "boodlers" in the underground slang of the time-- +were mostly technically skilled printers who had gone to the bad. +Many had once worked printing legitimate currency. +Boodlers operated in rings and gangs. Technical experts +engraved the bogus plates--commonly in basements in New York City. +Smooth confidence men passed large wads of high-quality, +high-denomination fakes, including the really sophisticated stuff-- +government bonds, stock certificates, and railway shares. +Cheaper, botched fakes were sold or sharewared to low-level +gangs of boodler wannabes. (The really cheesy lowlife boodlers +merely upgraded real bills by altering face values, +changing ones to fives, tens to hundreds, and so on.) + +The techniques of boodling were little-known and regarded +with a certain awe by the mid- nineteenth-century public. +The ability to manipulate the system for rip-off seemed +diabolically clever. As the skill and daring of the +boodlers increased, the situation became intolerable. +The federal government stepped in, and began offering +its own federal currency, which was printed in fancy green ink, +but only on the back--the original "greenbacks." And at first, +the improved security of the well-designed, well-printed +federal greenbacks seemed to solve the problem; but then +the counterfeiters caught on. Within a few years things were +worse than ever: a CENTRALIZED system where ALL security was bad! + +The local police were helpless. The Government tried offering +blood money to potential informants, but this met with little success. +Banks, plagued by boodling, gave up hope of police help and hired +private security men instead. Merchants and bankers queued up +by the thousands to buy privately-printed manuals on currency security, +slim little books like Laban Heath's INFALLIBLE GOVERNMENT +COUNTERFEIT DETECTOR. The back of the book offered Laban Heath's +patent microscope for five bucks. + +Then the Secret Service entered the picture. The first agents +were a rough and ready crew. Their chief was one William P. Wood, +a former guerilla in the Mexican War who'd won a reputation busting +contractor fraudsters for the War Department during the Civil War. +Wood, who was also Keeper of the Capital Prison, had a sideline +as a counterfeiting expert, bagging boodlers for the federal bounty money. + +Wood was named Chief of the new Secret Service in July 1865. +There were only ten Secret Service agents in all: Wood himself, +a handful who'd worked for him in the War Department, and a few +former private investigators--counterfeiting experts--whom Wood +had won over to public service. (The Secret Service of 1865 was +much the size of the Chicago Computer Fraud Task Force or the +Arizona Racketeering Unit of 1990.) These ten "Operatives" +had an additional twenty or so "Assistant Operatives" and "Informants." +Besides salary and per diem, each Secret Service employee received +a whopping twenty-five dollars for each boodler he captured. + +Wood himself publicly estimated that at least HALF of America's currency +was counterfeit, a perhaps pardonable perception. Within a year the +Secret Service had arrested over 200 counterfeiters. They busted about +two hundred boodlers a year for four years straight. + +Wood attributed his success to travelling fast and light, hitting the +bad-guys hard, and avoiding bureaucratic baggage. "Because my raids +were made without military escort and I did not ask the assistance +of state officers, I surprised the professional counterfeiter." + +Wood's social message to the once-impudent boodlers bore an eerie ring +of Sundevil: "It was also my purpose to convince such characters that +it would no longer be healthy for them to ply their vocation without +being handled roughly, a fact they soon discovered." + +William P. Wood, the Secret Service's guerilla pioneer, +did not end well. He succumbed to the lure of aiming for +the really big score. The notorious Brockway Gang of New York City, +headed by William E. Brockway, the "King of the Counterfeiters," +had forged a number of government bonds. They'd passed these +brilliant fakes on the prestigious Wall Street investment +firm of Jay Cooke and Company. The Cooke firm were frantic +and offered a huge reward for the forgers' plates. + +Laboring diligently, Wood confiscated the plates +(though not Mr. Brockway) and claimed the reward. +But the Cooke company treacherously reneged. +Wood got involved in a down-and-dirty lawsuit +with the Cooke capitalists. Wood's boss, +Secretary of the Treasury McCulloch, felt that +Wood's demands for money and glory were unseemly, +and even when the reward money finally came through, +McCulloch refused to pay Wood anything. +Wood found himself mired in a seemingly endless +round of federal suits and Congressional lobbying. + +Wood never got his money. And he lost his job to boot. +He resigned in 1869. + +Wood's agents suffered, too. On May 12, 1869, the second Chief +of the Secret Service took over, and almost immediately fired +most of Wood's pioneer Secret Service agents: Operatives, +Assistants and Informants alike. The practice of receiving $25 +per crook was abolished. And the Secret Service began the long, +uncertain process of thorough professionalization. + +Wood ended badly. He must have felt stabbed in the back. +In fact his entire organization was mangled. + +On the other hand, William P. Wood WAS the first head of the Secret Service. +William Wood was the pioneer. People still honor his name. Who remembers +the name of the SECOND head of the Secret Service? + +As for William Brockway (also known as "Colonel Spencer"), +he was finally arrested by the Secret Service in 1880. +He did five years in prison, got out, and was still boodling +at the age of seventy-four. + +# + +Anyone with an interest in Operation Sundevil-- +or in American computer-crime generally-- +could scarcely miss the presence of Gail Thackeray, +Assistant Attorney General of the State of Arizona. +Computer-crime training manuals often cited +Thackeray's group and her work; she was the +highest-ranking state official to specialize +in computer-related offenses. Her name had been +on the Sundevil press release (though modestly ranked +well after the local federal prosecuting attorney and +the head of the Phoenix Secret Service office). + +As public commentary, and controversy, began to mount +about the Hacker Crackdown, this Arizonan state official +began to take a higher and higher public profile. +Though uttering almost nothing specific about +the Sundevil operation itself, she coined some +of the most striking soundbites of the growing propaganda war: +"Agents are operating in good faith, and I don't think +you can say that for the hacker community," was one. +Another was the memorable "I am not a mad dog prosecutor" +(Houston Chronicle, Sept 2, 1990.) In the meantime, +the Secret Service maintained its usual extreme discretion; +the Chicago Unit, smarting from the backlash +of the Steve Jackson scandal, had gone completely to earth. + +As I collated my growing pile of newspaper clippings, +Gail Thackeray ranked as a comparative fount of public +knowledge on police operations. + +I decided that I had to get to know Gail Thackeray. +I wrote to her at the Arizona Attorney General's Office. +Not only did she kindly reply to me, but, to my astonishment, +she knew very well what "cyberpunk" science fiction was. + +Shortly after this, Gail Thackeray lost her job. +And I temporarily misplaced my own career as +a science-fiction writer, to become a full-time +computer-crime journalist. In early March, 1991, +I flew to Phoenix, Arizona, to interview Gail Thackeray +for my book on the hacker crackdown. + +# + +"Credit cards didn't used to cost anything to get," +says Gail Thackeray. "Now they cost forty bucks-- +and that's all just to cover the costs from RIP-OFF ARTISTS." + +Electronic nuisance criminals are parasites. +One by one they're not much harm, no big deal. +But they never come just one by one. They come in swarms, +heaps, legions, sometimes whole subcultures. And they bite. +Every time we buy a credit card today, we lose a little financial +vitality to a particular species of bloodsucker. + +What, in her expert opinion, are the worst forms of electronic crime, +I ask, consulting my notes. Is it--credit card fraud? Breaking into +ATM bank machines? Phone-phreaking? Computer intrusions? +Software viruses? Access-code theft? Records tampering? +Software piracy? Pornographic bulletin boards? +Satellite TV piracy? Theft of cable service? +It's a long list. By the time I reach the end +of it I feel rather depressed. + +"Oh no," says Gail Thackeray, leaning forward over the table, +her whole body gone stiff with energetic indignation, +"the biggest damage is telephone fraud. Fake sweepstakes, +fake charities. Boiler-room con operations. You could pay off +the national debt with what these guys steal. . . . +They target old people, they get hold of credit ratings +and demographics, they rip off the old and the weak." +The words come tumbling out of her. + +It's low-tech stuff, your everyday boiler-room fraud. +Grifters, conning people out of money over the phone, +have been around for decades. This is where the word "phony" came from! + +It's just that it's so much EASIER now, horribly facilitated by advances +in technology and the byzantine structure of the modern phone system. +The same professional fraudsters do it over and over, Thackeray tells me, +they hide behind dense onion-shells of fake companies. . . fake holding +corporations nine or ten layers deep, registered all over the map. +They get a phone installed under a false name in an empty safe-house. +And then they call-forward everything out of that phone to yet +another phone, a phone that may even be in another STATE. +And they don't even pay the charges on their phones; +after a month or so, they just split; set up somewhere else +in another Podunkville with the same seedy crew of veteran phone-crooks. +They buy or steal commercial credit card reports, slap them on the PC, +have a program pick out people over sixty-five who pay a lot to charities. +A whole subculture living off this, merciless folks on the con. + +"The `light-bulbs for the blind' people," Thackeray muses, +with a special loathing. "There's just no end to them." + +We're sitting in a downtown diner in Phoenix, Arizona. +It's a tough town, Phoenix. A state capital seeing some hard times. +Even to a Texan like myself, Arizona state politics seem rather baroque. +There was, and remains, endless trouble over the Martin Luther King holiday, +the sort of stiff-necked, foot-shooting incident for which Arizona politics +seem famous. There was Evan Mecham, the eccentric Republican millionaire +governor who was impeached, after reducing state government to a +ludicrous shambles. Then there was the national Keating scandal, +involving Arizona savings and loans, in which both of Arizona's +U.S. senators, DeConcini and McCain, played sadly prominent roles. + +And the very latest is the bizarre AzScam case, +in which state legislators were videotaped, +eagerly taking cash from an informant of the Phoenix city +police department, who was posing as a Vegas mobster. + +"Oh," says Thackeray cheerfully. "These people are amateurs here, +they thought they were finally getting to play with the big boys. +They don't have the least idea how to take a bribe! +It's not institutional corruption. It's not like back in Philly." + +Gail Thackeray was a former prosecutor in Philadelphia. +Now she's a former assistant attorney general of the State of Arizona. +Since moving to Arizona in 1986, she had worked under the aegis +of Steve Twist, her boss in the Attorney General's office. +Steve Twist wrote Arizona's pioneering computer crime laws +and naturally took an interest in seeing them enforced. +It was a snug niche, and Thackeray's Organized Crime and +Racketeering Unit won a national reputation for ambition +and technical knowledgeability. . . . Until the latest +election in Arizona. Thackeray's boss ran for the top +job, and lost. The victor, the new Attorney General, +apparently went to some pains to eliminate the bureaucratic +traces of his rival, including his pet group--Thackeray's group. +Twelve people got their walking papers. + +Now Thackeray's painstakingly assembled computer lab +sits gathering dust somewhere in the glass-and-concrete +Attorney General's HQ on 1275 Washington Street. +Her computer-crime books, her painstakingly garnered +back issues of phreak and hacker zines, all bought +at her own expense--are piled in boxes somewhere. +The State of Arizona is simply not particularly +interested in electronic racketeering at the moment. + +At the moment of our interview, Gail Thackeray, +officially unemployed, is working out of the county +sheriff's office, living on her savings, and prosecuting +several cases--working 60-hour weeks, just as always-- +for no pay at all. "I'm trying to train people," +she mutters. + +Half her life seems to be spent training people--merely pointing out, +to the naive and incredulous (such as myself) that this stuff +is ACTUALLY GOING ON OUT THERE. It's a small world, computer crime. +A young world. Gail Thackeray, a trim blonde Baby-Boomer who favors +Grand Canyon white-water rafting to kill some slow time, +is one of the world's most senior, most veteran "hacker-trackers." +Her mentor was Donn Parker, the California think-tank theorist +who got it all started `way back in the mid-70s, the "grandfather +of the field," "the great bald eagle of computer crime." + +And what she has learned, Gail Thackeray teaches. Endlessly. +Tirelessly. To anybody. To Secret Service agents and state police, +at the Glynco, Georgia federal training center. To local police, +on "roadshows" with her slide projector and notebook. +To corporate security personnel. To journalists. To parents. + +Even CROOKS look to Gail Thackeray for advice. +Phone-phreaks call her at the office. They know very +well who she is. They pump her for information +on what the cops are up to, how much they know. +Sometimes whole CROWDS of phone phreaks, +hanging out on illegal conference calls, will call Gail +Thackeray up. They taunt her. And, as always, +they boast. Phone-phreaks, real stone phone-phreaks, +simply CANNOT SHUT UP. They natter on for hours. + +Left to themselves, they mostly talk about the intricacies +of ripping-off phones; it's about as interesting as listening +to hot-rodders talk about suspension and distributor-caps. +They also gossip cruelly about each other. And when talking +to Gail Thackeray, they incriminate themselves. "I have tapes," +Thackeray says coolly. + +Phone phreaks just talk like crazy. "Dial-Tone" out in Alabama +has been known to spend half-an-hour simply reading stolen +phone-codes aloud into voice-mail answering machines. +Hundreds, thousands of numbers, recited in a monotone, +without a break--an eerie phenomenon. When arrested, +it's a rare phone phreak who doesn't inform at endless length +on everybody he knows. + +Hackers are no better. What other group of criminals, +she asks rhetorically, publishes newsletters and holds conventions? +She seems deeply nettled by the sheer brazenness of this behavior, +though to an outsider, this activity might make one wonder +whether hackers should be considered "criminals" at all. +Skateboarders have magazines, and they trespass a lot. +Hot rod people have magazines and they break speed limits +and sometimes kill people. . . . + +I ask her whether it would be any loss to society if phone phreaking +and computer hacking, as hobbies, simply dried up and blew away, +so that nobody ever did it again. + +She seems surprised. "No," she says swiftly. "Maybe a little. . . +in the old days. . .the MIT stuff. . . . But there's a lot of wonderful, +legal stuff you can do with computers now, you don't have to break into +somebody else's just to learn. You don't have that excuse. +You can learn all you like." + +Did you ever hack into a system? I ask. + +The trainees do it at Glynco. Just to demonstrate system vulnerabilities. +She's cool to the notion. Genuinely indifferent. + +"What kind of computer do you have?" + +"A Compaq 286LE," she mutters. + +"What kind do you WISH you had?" + +At this question, the unmistakable light of true hackerdom flares in +Gail Thackeray's eyes. She becomes tense, animated, the words pour out: +"An Amiga 2000 with an IBM card and Mac emulation! The most common hacker +machines are Amigas and Commodores. And Apples." If she had the Amiga, +she enthuses, she could run a whole galaxy of seized computer-evidence disks +on one convenient multifunctional machine. A cheap one, too. Not like the +old Attorney General lab, where they had an ancient CP/M machine, +assorted Amiga flavors and Apple flavors, a couple IBMS, all the +utility software. . .but no Commodores. The workstations down +at the Attorney General's are Wang dedicated word-processors. +Lame machines tied in to an office net--though at least they get +on- line to the Lexis and Westlaw legal data services. + +I don't say anything. I recognize the syndrome, though. +This computer-fever has been running through segments of +our society for years now. It's a strange kind of lust: +K-hunger, Meg-hunger; but it's a shared disease; +it can kill parties dead, as conversation spirals into +the deepest and most deviant recesses of software releases +and expensive peripherals. . . . The mark of the hacker beast. +I have it too. The whole "electronic community," whatever the hell +that is, has it. Gail Thackeray has it. Gail Thackeray is a hacker cop. +My immediate reaction is a strong rush of indignant pity: +WHY DOESN'T SOMEBODY BUY THIS WOMAN HER AMIGA?! +It's not like she's asking for a Cray X-MP +supercomputer mainframe; an Amiga's a sweet little +cookie-box thing. We're losing zillions in organized fraud; +prosecuting and defending a single hacker case in court can cost +a hundred grand easy. How come nobody can come up with four lousy grand +so this woman can do her job? For a hundred grand we could buy every +computer cop in America an Amiga. There aren't that many of 'em. + +Computers. The lust, the hunger, for computers. +The loyalty they inspire, the intense sense of possessiveness. +The culture they have bred. I myself am sitting in downtown Phoenix, +Arizona because it suddenly occurred to me that the police might-- +just MIGHT--come and take away my computer. The prospect of this, +the mere IMPLIED THREAT, was unbearable. It literally changed my life. +It was changing the lives of many others. Eventually it would change +everybody's life. + +Gail Thackeray was one of the top computer-crime people in America. +And I was just some novelist, and yet I had a better computer than hers. +PRACTICALLY EVERYBODY I KNEW had a better computer than Gail Thackeray +and her feeble laptop 286. It was like sending the sheriff in to clean +up Dodge City and arming her with a slingshot cut from an old rubber tire. + +But then again, you don't need a howitzer to enforce the law. +You can do a lot just with a badge. With a badge alone, +you can basically wreak havoc, take a terrible vengeance on wrongdoers. +Ninety percent of "computer crime investigation" is just "crime investigation:" +names, places, dossiers, modus operandi, search warrants, victims, +complainants, informants. . . . + +What will computer crime look like in ten years? Will it get better? +Did "Sundevil" send 'em reeling back in confusion? + +It'll be like it is now, only worse, she tells me with perfect conviction. +Still there in the background, ticking along, changing with the times: +the criminal underworld. It'll be like drugs are. Like our problems +with alcohol. All the cops and laws in the world never solved our problems +with alcohol. If there's something people want, a certain percentage +of them are just going to take it. Fifteen percent of the populace +will never steal. Fifteen percent will steal most anything not nailed down. +The battle is for the hearts and minds of the remaining seventy percent. + +And criminals catch on fast. If there's not "too steep a learning curve"-- +if it doesn't require a baffling amount of expertise and practice-- +then criminals are often some of the first through the gate of a +new technology. Especially if it helps them to hide. +They have tons of cash, criminals. The new communications tech-- +like pagers, cellular phones, faxes, Federal Express--were pioneered +by rich corporate people, and by criminals. In the early years +of pagers and beepers, dope dealers were so enthralled this technology +that owing a beeper was practically prima facie evidence of cocaine dealing. +CB radio exploded when the speed limit hit 55 and breaking the highway law +became a national pastime. Dope dealers send cash by Federal Express, +despite, or perhaps BECAUSE OF, the warnings in FedEx offices that tell you +never to try this. Fed Ex uses X-rays and dogs on their mail, +to stop drug shipments. That doesn't work very well. + +Drug dealers went wild over cellular phones. +There are simple methods of faking ID on cellular phones, +making the location of the call mobile, free of charge, +and effectively untraceable. Now victimized cellular +companies routinely bring in vast toll-lists of calls +to Colombia and Pakistan. + +Judge Greene's fragmentation of the phone company +is driving law enforcement nuts. Four thousand +telecommunications companies. Fraud skyrocketing. +Every temptation in the world available with a phone +and a credit card number. Criminals untraceable. +A galaxy of "new neat rotten things to do." + +If there were one thing Thackeray would like to have, +it would be an effective legal end-run through this new +fragmentation minefield. + +It would be a new form of electronic search warrant, +an "electronic letter of marque" to be issued by a judge. +It would create a new category of "electronic emergency." +Like a wiretap, its use would be rare, but it would cut +across state lines and force swift cooperation from all concerned. +Cellular, phone, laser, computer network, PBXes, AT&T, Baby Bells, +long-distance entrepreneurs, packet radio. Some document, +some mighty court-order, that could slice through four thousand +separate forms of corporate red-tape, and get her at once to +the source of calls, the source of email threats and viruses, +the sources of bomb threats, kidnapping threats. "From now on," +she says, "the Lindbergh baby will always die." + +Something that would make the Net sit still, if only for a moment. +Something that would get her up to speed. Seven league boots. +That's what she really needs. "Those guys move in nanoseconds +and I'm on the Pony Express." + +And then, too, there's the coming international angle. +Electronic crime has never been easy to localize, +to tie to a physical jurisdiction. And phone-phreaks +and hackers loathe boundaries, they jump them whenever they can. +The English. The Dutch. And the Germans, especially the ubiquitous +Chaos Computer Club. The Australians. They've all learned phone-phreaking +from America. It's a growth mischief industry. The multinational +networks are global, but governments and the police simply aren't. +Neither are the laws. Or the legal frameworks for citizen protection. + +One language is global, though--English. Phone phreaks speak English; +it's their native tongue even if they're Germans. English may have started +in England but now it's the Net language; it might as well be called "CNNese." + +Asians just aren't much into phone phreaking. They're the world masters +at organized software piracy. The French aren't into phone-phreaking either. +The French are into computerized industrial espionage. + +In the old days of the MIT righteous hackerdom, crashing systems +didn't hurt anybody. Not all that much, anyway. Not permanently. +Now the players are more venal. Now the consequences are worse. +Hacking will begin killing people soon. Already there are methods +of stacking calls onto 911 systems, annoying the police, and possibly +causing the death of some poor soul calling in with a genuine emergency. +Hackers in Amtrak computers, or air-traffic control computers, will kill +somebody someday. Maybe a lot of people. Gail Thackeray expects it. + +And the viruses are getting nastier. The "Scud" virus is the latest one out. +It wipes hard-disks. + +According to Thackeray, the idea that phone-phreaks are Robin Hoods is a fraud. +They don't deserve this repute. Basically, they pick on the weak. AT&T now +protects itself with the fearsome ANI (Automatic Number Identification) +trace capability. When AT&T wised up and tightened security generally, +the phreaks drifted into the Baby Bells. The Baby Bells lashed out in 1989 +and 1990, so the phreaks switched to smaller long-distance entrepreneurs. +Today, they are moving into locally owned PBXes and voice-mail systems, +which are full of security holes, dreadfully easy to hack. These victims +aren't the moneybags Sheriff of Nottingham or Bad King John, but small groups +of innocent people who find it hard to protect themselves, and who really +suffer from these depredations. Phone phreaks pick on the weak. They do it +for power. If it were legal, they wouldn't do it. They don't want service, +or knowledge, they want the thrill of power-tripping. There's plenty of +knowledge or service around if you're willing to pay. Phone phreaks don't pay, +they steal. It's because it is illegal that it feels like power, +that it gratifies their vanity. + +I leave Gail Thackeray with a handshake at the door of her office building-- +a vast International-Style office building downtown. The Sheriff's office +is renting part of it. I get the vague impression that quite a lot of the +building is empty--real estate crash. + +In a Phoenix sports apparel store, in a downtown mall, I meet +the "Sun Devil" himself. He is the cartoon mascot of +Arizona State University, whose football stadium, "Sundevil," +is near the local Secret Service HQ--hence the name Operation Sundevil. +The Sun Devil himself is named "Sparky." Sparky the Sun Devil is maroon +and bright yellow, the school colors. Sparky brandishes a three-tined +yellow pitchfork. He has a small mustache, pointed ears, a barbed tail, +and is dashing forward jabbing the air with the pitchfork, +with an expression of devilish glee. + +Phoenix was the home of Operation Sundevil. The Legion of Doom +ran a hacker bulletin board called "The Phoenix Project." +An Australian hacker named "Phoenix" once burrowed through +the Internet to attack Cliff Stoll, then bragged and boasted +about it to The New York Times. This net of coincidence +is both odd and meaningless. + +The headquarters of the Arizona Attorney General, Gail Thackeray's +former workplace, is on 1275 Washington Avenue. Many of the downtown +streets in Phoenix are named after prominent American presidents: +Washington, Jefferson, Madison. . . . + +After dark, all the employees go home to their suburbs. +Washington, Jefferson and Madison--what would be the +Phoenix inner city, if there were an inner city in this +sprawling automobile-bred town--become the haunts +of transients and derelicts. The homeless. The sidewalks +along Washington are lined with orange trees. +Ripe fallen fruit lies scattered like croquet balls +on the sidewalks and gutters. No one seems to be eating them. +I try a fresh one. It tastes unbearably bitter. + +The Attorney General's office, built in 1981 during the +Babbitt administration, is a long low two-story building +of white cement and wall-sized sheets of curtain-glass. +Behind each glass wall is a lawyer's office, quite open +and visible to anyone strolling by. Across the street +is a dour government building labelled simply ECONOMIC SECURITY, +something that has not been in great supply in the American +Southwest lately. + +The offices are about twelve feet square. They feature +tall wooden cases full of red-spined lawbooks; +Wang computer monitors; telephones; Post-it notes galore. +Also framed law diplomas and a general excess of bad +Western landscape art. Ansel Adams photos are a big favorite, +perhaps to compensate for the dismal specter of the parking lot, +two acres of striped black asphalt, which features gravel landscaping +and some sickly-looking barrel cacti. + +It has grown dark. Gail Thackeray has told me that the people +who work late here, are afraid of muggings in the parking lot. +It seems cruelly ironic that a woman tracing electronic racketeers +across the interstate labyrinth of Cyberspace should fear an assault +by a homeless derelict in the parking lot of her own workplace. + +Perhaps this is less than coincidence. Perhaps these two seemingly +disparate worlds are somehow generating one another. The poor and +disenfranchised take to the streets, while the rich and computer-equipped, +safe in their bedrooms, chatter over their modems. Quite often the derelicts +kick the glass out and break in to the lawyers' offices, if they see something +they need or want badly enough. + +I cross the parking lot to the street behind the Attorney General's office. +A pair of young tramps are bedding down on flattened sheets of cardboard, +under an alcove stretching over the sidewalk. One tramp wears a +glitter-covered T-shirt reading "CALIFORNIA" in Coca-Cola cursive. +His nose and cheeks look chafed and swollen; they glisten with +what seems to be Vaseline. The other tramp has a ragged long-sleeved +shirt and lank brown hair parted in the middle. They both wear blue jeans +coated in grime. They are both drunk. + +"You guys crash here a lot?" I ask them. + +They look at me warily. I am wearing black jeans, a black pinstriped +suit jacket and a black silk tie. I have odd shoes and a funny haircut. + +"It's our first time here," says the red-nosed tramp unconvincingly. +There is a lot of cardboard stacked here. More than any two people could use. + +"We usually stay at the Vinnie's down the street," says the brown-haired tramp, +puffing a Marlboro with a meditative air, as he sprawls with his head on +a blue nylon backpack. "The Saint Vincent's." + +"You know who works in that building over there?" I ask, pointing. + +The brown-haired tramp shrugs. "Some kind of attorneys, it says." + +We urge one another to take it easy. I give them five bucks. + +A block down the street I meet a vigorous workman who is wheeling along +some kind of industrial trolley; it has what appears to be a tank of +propane on it. + +We make eye contact. We nod politely. I walk past him. "Hey! +Excuse me sir!" he says. + +"Yes?" I say, stopping and turning. + +"Have you seen," the guy says rapidly, "a black guy, about 6'7", +scars on both his cheeks like this--" he gestures-- "wears a +black baseball cap on backwards, wandering around here anyplace?" + +"Sounds like I don't much WANT to meet him," I say. + +"He took my wallet," says my new acquaintance. +"Took it this morning. Y'know, some people would be +SCARED of a guy like that. But I'm not scared. +I'm from Chicago. I'm gonna hunt him down. +We do things like that in Chicago." + +"Yeah?" + +"I went to the cops and now he's got an APB out on his ass," +he says with satisfaction. "You run into him, you let me know." + +"Okay," I say. "What is your name, sir?" + +"Stanley. . . ." + +"And how can I reach you?" + +"Oh," Stanley says, in the same rapid voice, +"you don't have to reach, uh, me. +You can just call the cops. Go straight to the cops." +He reaches into a pocket and pulls out a greasy piece of pasteboard. +"See, here's my report on him." + +I look. The "report," the size of an index card, is labelled PRO-ACT: +Phoenix Residents Opposing Active Crime Threat. . . . or is it +Organized Against Crime Threat? In the darkening street it's hard +to read. Some kind of vigilante group? Neighborhood watch? +I feel very puzzled. + +"Are you a police officer, sir?" + +He smiles, seems very pleased by the question. + +"No," he says. + +"But you are a `Phoenix Resident?'" + +"Would you believe a homeless person," Stanley says. + +"Really? But what's with the. . . ." For the first time I take a close look +at Stanley's trolley. It's a rubber-wheeled thing of industrial metal, +but the device I had mistaken for a tank of propane is in fact a water-cooler. +Stanley also has an Army duffel-bag, stuffed tight as a sausage with clothing +or perhaps a tent, and, at the base of his trolley, a cardboard box and a +battered leather briefcase. + +"I see," I say, quite at a loss. For the first time I notice that Stanley +has a wallet. He has not lost his wallet at all. It is in his back pocket +and chained to his belt. It's not a new wallet. It seems to have seen +a lot of wear. + +"Well, you know how it is, brother," says Stanley. +Now that I know that he is homeless--A POSSIBLE +THREAT--my entire perception of him has changed +in an instant. His speech, which once seemed just +bright and enthusiastic, now seems to have a +dangerous tang of mania. "I have to do this!" +he assures me. "Track this guy down. . . . +It's a thing I do. . . you know. . .to keep myself together!" +He smiles, nods, lifts his trolley by its decaying rubber handgrips. + +"Gotta work together, y'know," Stanley booms, his face alight +with cheerfulness, "the police can't do everything!" +The gentlemen I met in my stroll in downtown Phoenix +are the only computer illiterates in this book. +To regard them as irrelevant, however, would be a grave mistake. + +As computerization spreads across society, the populace at large +is subjected to wave after wave of future shock. But, as a +necessary converse, the "computer community" itself is subjected +to wave after wave of incoming computer illiterates. +How will those currently enjoying America's digital bounty regard, +and treat, all this teeming refuse yearning to breathe free? +Will the electronic frontier be another Land of Opportunity-- +or an armed and monitored enclave, where the disenfranchised +snuggle on their cardboard at the locked doors of our houses of justice? + +Some people just don't get along with computers. They can't read. +They can't type. They just don't have it in their heads to master +arcane instructions in wirebound manuals. Somewhere, the process +of computerization of the populace will reach a limit. Some people-- +quite decent people maybe, who might have thrived in any other situation-- +will be left irretrievably outside the bounds. What's to be done with +these people, in the bright new shiny electroworld? How will they +be regarded, by the mouse-whizzing masters of cyberspace? With contempt? +Indifference? Fear? + +In retrospect, it astonishes me to realize how quickly poor Stanley +became a perceived threat. Surprise and fear are closely allied feelings. +And the world of computing is full of surprises. + +I met one character in the streets of Phoenix whose role in this book +is supremely and directly relevant. That personage was Stanley's giant +thieving scarred phantom. This phantasm is everywhere in this book. +He is the specter haunting cyberspace. + +Sometimes he's a maniac vandal ready to smash the phone system +for no sane reason at all. Sometimes he's a fascist fed, +coldly programming his mighty mainframes to destroy our Bill of Rights. +Sometimes he's a telco bureaucrat, covertly conspiring to register all modems +in the service of an Orwellian surveillance regime. Mostly, though, +this fearsome phantom is a "hacker." He's strange, he doesn't belong, +he's not authorized, he doesn't smell right, he's not keeping his proper place, +he's not one of us. The focus of fear is the hacker, for much the same +reasons that Stanley's fancied assailant is black. + +Stanley's demon can't go away, because he doesn't exist. +Despite singleminded and tremendous effort, he can't be arrested, +sued, jailed, or fired. The only constructive way to do ANYTHING +about him is to learn more about Stanley himself. This learning process +may be repellent, it may be ugly, it may involve grave elements of paranoiac +confusion, but it's necessary. Knowing Stanley requires something more +than class-crossing condescension. It requires more than steely +legal objectivity. It requires human compassion and sympathy. + +To know Stanley is to know his demon. If you know the other guy's demon, +then maybe you'll come to know some of your own. You'll be able to +separate reality from illusion. And then you won't do your cause, +and yourself, more harm than good. Like poor damned Stanley from Chicago did. + +# + +The Federal Computer Investigations Committee (FCIC) is the most important +and influential organization in the realm of American computer-crime. +Since the police of other countries have largely taken their computer-crime +cues from American methods, the FCIC might well be called the most important +computer crime group in the world. + +It is also, by federal standards, an organization of great unorthodoxy. +State and local investigators mix with federal agents. Lawyers, +financial auditors and computer-security programmers trade notes +with street cops. Industry vendors and telco security people show up +to explain their gadgetry and plead for protection and justice. +Private investigators, think-tank experts and industry pundits throw in +their two cents' worth. The FCIC is the antithesis of a formal bureaucracy. + +Members of the FCIC are obscurely proud of this fact; they recognize their +group as aberrant, but are entirely convinced that this, for them, +outright WEIRD behavior is nevertheless ABSOLUTELY NECESSARY +to get their jobs done. + +FCIC regulars --from the Secret Service, the FBI, the IRS, +the Department of Labor, the offices of federal attorneys, +state police, the Air Force, from military intelligence-- +often attend meetings, held hither and thither across the country, +at their own expense. The FCIC doesn't get grants. It doesn't +charge membership fees. It doesn't have a boss. It has no headquarters-- +just a mail drop in Washington DC, at the Fraud Division of the Secret Service. +It doesn't have a budget. It doesn't have schedules. It meets three times +a year--sort of. Sometimes it issues publications, but the FCIC +has no regular publisher, no treasurer, not even a secretary. +There are no minutes of FCIC meetings. Non-federal people are considered +"non-voting members," but there's not much in the way of elections. +There are no badges, lapel pins or certificates of membership. +Everyone is on a first-name basis. There are about forty of them. +Nobody knows how many, exactly. People come, people go-- +sometimes people "go" formally but still hang around anyway. +Nobody has ever exactly figured out what "membership" of this +"Committee" actually entails. + +Strange as this may seem to some, to anyone familiar with the social world +of computing, the "organization" of the FCIC is very recognizable. + +For years now, economists and management theorists have speculated +that the tidal wave of the information revolution would destroy rigid, +pyramidal bureaucracies, where everything is top-down and +centrally controlled. Highly trained "employees" would take on +much greater autonomy, being self-starting, and self-motivating, +moving from place to place, task to task, with great speed and fluidity. +"Ad-hocracy" would rule, with groups of people spontaneously knitting +together across organizational lines, tackling the problem at hand, +applying intense computer-aided expertise to it, and then vanishing +whence they came. + +This is more or less what has actually happened in the world of +federal computer investigation. With the conspicuous exception +of the phone companies, which are after all over a hundred years old, +practically EVERY organization that plays any important role in this book +functions just like the FCIC. The Chicago Task Force, the Arizona +Racketeering Unit, the Legion of Doom, the Phrack crowd, the +Electronic Frontier Foundation--they ALL look and act like "tiger teams" +or "user's groups." They are all electronic ad-hocracies leaping up +spontaneously to attempt to meet a need. + +Some are police. Some are, by strict definition, criminals. +Some are political interest-groups. But every single group +has that same quality of apparent spontaneity--"Hey, gang! +My uncle's got a barn--let's put on a show!" + +Every one of these groups is embarrassed by this "amateurism," +and, for the sake of their public image in a world of non-computer people, +they all attempt to look as stern and formal and impressive as possible. +These electronic frontier-dwellers resemble groups of nineteenth-century +pioneers hankering after the respectability of statehood. +There are however, two crucial differences in the historical experience +of these "pioneers" of the nineteeth and twenty-first centuries. + +First, powerful information technology DOES play into the hands of small, +fluid, loosely organized groups. There have always been "pioneers," +"hobbyists," "amateurs," "dilettantes," "volunteers," "movements," +"users' groups" and "blue-ribbon panels of experts" around. +But a group of this kind--when technically equipped to ship +huge amounts of specialized information, at lightning speed, +to its members, to government, and to the press--is simply +a different kind of animal. It's like the difference between +an eel and an electric eel. + +The second crucial change is that American society is currently +in a state approaching permanent technological revolution. +In the world of computers particularly, it is practically impossible +to EVER stop being a "pioneer," unless you either drop dead or +deliberately jump off the bus. The scene has never slowed down +enough to become well-institutionalized. And after twenty, thirty, +forty years the "computer revolution" continues to spread, +to permeate new corners of society. Anything that really works +is already obsolete. + +If you spend your entire working life as a "pioneer," the word "pioneer" +begins to lose its meaning. Your way of life looks less and less like +an introduction to something else" more stable and organized, +and more and more like JUST THE WAY THINGS ARE. A "permanent revolution" +is really a contradiction in terms. If "turmoil" lasts long enough, +it simply becomes A NEW KIND OF SOCIETY--still the same game of history, +but new players, new rules. + +Apply this to the world of late twentieth-century law enforcement, +and the implications are novel and puzzling indeed. Any bureaucratic +rulebook you write about computer-crime will be flawed when you write it, +and almost an antique by the time it sees print. The fluidity and fast +reactions of the FCIC give them a great advantage in this regard, +which explains their success. Even with the best will in the world +(which it does not, in fact, possess) it is impossible for an organization +the size of the U.S. Federal Bureau of Investigation to get up to speed +on the theory and practice of computer crime. If they tried to train all +their agents to do this, it would be SUICIDAL, as they would NEVER BE ABLE +TO DO ANYTHING ELSE. + +The FBI does try to train its agents in the basics of electronic crime, +at their base in Quantico, Virginia. And the Secret Service, along with +many other law enforcement groups, runs quite successful and well-attended +training courses on wire fraud, business crime, and computer intrusion +at the Federal Law Enforcement Training Center (FLETC, pronounced "fletsy") +in Glynco, Georgia. But the best efforts of these bureaucracies does not +remove the absolute need for a "cutting-edge mess" like the FCIC. + +For you see--the members of FCIC ARE the trainers of the rest +of law enforcement. Practically and literally speaking, +they are the Glynco computer-crime faculty by another name. +If the FCIC went over a cliff on a bus, the U.S. law enforcement +community would be rendered deaf dumb and blind in the world +of computer crime, and would swiftly feel a desperate need +to reinvent them. And this is no time to go starting from scratch. + +On June 11, 1991, I once again arrived in Phoenix, Arizona, +for the latest meeting of the Federal Computer Investigations Committee. +This was more or less the twentieth meeting of this stellar group. +The count was uncertain, since nobody could figure out whether to +include the meetings of "the Colluquy," which is what the FCIC +was called in the mid-1980s before it had even managed to obtain +the dignity of its own acronym. + +Since my last visit to Arizona, in May, the local AzScam bribery scandal +had resolved itself in a general muddle of humiliation. The Phoenix chief +of police, whose agents had videotaped nine state legislators up to no good, +had resigned his office in a tussle with the Phoenix city council over +the propriety of his undercover operations. + +The Phoenix Chief could now join Gail Thackeray and eleven of her closest +associates in the shared experience of politically motivated unemployment. +As of June, resignations were still continuing at the Arizona Attorney +General's office, which could be interpreted as either a New Broom +Sweeping Clean or a Night of the Long Knives Part II, depending on +your point of view. + +The meeting of FCIC was held at the Scottsdale Hilton Resort. +Scottsdale is a wealthy suburb of Phoenix, known as "Scottsdull" +to scoffing local trendies, but well-equipped with posh shopping-malls +and manicured lawns, while conspicuously undersupplied with homeless derelicts. +The Scottsdale Hilton Resort was a sprawling hotel in postmodern +crypto-Southwestern style. It featured a "mission bell tower" +plated in turquoise tile and vaguely resembling a Saudi minaret. + +Inside it was all barbarically striped Santa Fe Style decor. +There was a health spa downstairs and a large oddly-shaped +pool in the patio. A poolside umbrella-stand offered Ben and Jerry's +politically correct Peace Pops. + +I registered as a member of FCIC, attaining a handy discount rate, +then went in search of the Feds. Sure enough, at the back of the +hotel grounds came the unmistakable sound of Gail Thackeray +holding forth. + +Since I had also attended the Computers Freedom and Privacy conference +(about which more later), this was the second time I had seen Thackeray +in a group of her law enforcement colleagues. Once again I was struck +by how simply pleased they seemed to see her. It was natural that she'd +get SOME attention, as Gail was one of two women in a group of some thirty men; +but there was a lot more to it than that. + +Gail Thackeray personifies the social glue of the FCIC. They could give +a damn about her losing her job with the Attorney General. They were sorry +about it, of course, but hell, they'd all lost jobs. If they were the kind +of guys who liked steady boring jobs, they would never have gotten into +computer work in the first place. + +I wandered into her circle and was immediately introduced to five strangers. +The conditions of my visit at FCIC were reviewed. I would not quote +anyone directly. I would not tie opinions expressed to the agencies +of the attendees. I would not (a purely hypothetical example) +report the conversation of a guy from the Secret Service talking +quite civilly to a guy from the FBI, as these two agencies NEVER +talk to each other, and the IRS (also present, also hypothetical) +NEVER TALKS TO ANYBODY. + +Worse yet, I was forbidden to attend the first conference. And I didn't. +I have no idea what the FCIC was up to behind closed doors that afternoon. +I rather suspect that they were engaging in a frank and thorough confession +of their errors, goof-ups and blunders, as this has been a feature of every +FCIC meeting since their legendary Memphis beer-bust of 1986. Perhaps the +single greatest attraction of FCIC is that it is a place where you can go, +let your hair down, and completely level with people who actually comprehend +what you are talking about. Not only do they understand you, but they +REALLY PAY ATTENTION, they are GRATEFUL FOR YOUR INSIGHTS, and they +FORGIVE YOU, which in nine cases out of ten is something even your +boss can't do, because as soon as you start talking "ROM," "BBS," +or "T-1 trunk," his eyes glaze over. + +I had nothing much to do that afternoon. The FCIC were beavering away +in their conference room. Doors were firmly closed, windows too dark +to peer through. I wondered what a real hacker, a computer intruder, +would do at a meeting like this. + +The answer came at once. He would "trash" the place. Not reduce the place +to trash in some orgy of vandalism; that's not the use of the term in the +hacker milieu. No, he would quietly EMPTY THE TRASH BASKETS and silently +raid any valuable data indiscreetly thrown away. + +Journalists have been known to do this. (Journalists hunting information +have been known to do almost every single unethical thing that hackers +have ever done. They also throw in a few awful techniques all their own.) +The legality of `trashing' is somewhat dubious but it is not in fact +flagrantly illegal. It was, however, absurd to contemplate trashing the FCIC. +These people knew all about trashing. I wouldn't last fifteen seconds. + +The idea sounded interesting, though. I'd been hearing a lot about +the practice lately. On the spur of the moment, I decided I would try +trashing the office ACROSS THE HALL from the FCIC, an area which had +nothing to do with the investigators. + +The office was tiny; six chairs, a table. . . . Nevertheless, it was open, +so I dug around in its plastic trash can. + +To my utter astonishment, I came up with the torn scraps of a SPRINT +long-distance phone bill. More digging produced a bank statement +and the scraps of a hand-written letter, along with gum, cigarette ashes, +candy wrappers and a day-old-issue of USA TODAY. + +The trash went back in its receptacle while the scraps of data went into +my travel bag. I detoured through the hotel souvenir shop for some +Scotch tape and went up to my room. + +Coincidence or not, it was quite true. Some poor soul had, in fact, +thrown a SPRINT bill into the hotel's trash. Date May 1991, +total amount due: $252.36. Not a business phone, either, +but a residential bill, in the name of someone called Evelyn +(not her real name). Evelyn's records showed a ## PAST DUE BILL ##! +Here was her nine-digit account ID. Here was a stern computer-printed warning: + +"TREAT YOUR FONCARD AS YOU WOULD ANY CREDIT CARD. TO SECURE AGAINST FRAUD, +NEVER GIVE YOUR FONCARD NUMBER OVER THE PHONE UNLESS YOU INITIATED THE CALL. +IF YOU RECEIVE SUSPICIOUS CALLS PLEASE NOTIFY CUSTOMER SERVICE IMMEDIATELY!" + +I examined my watch. Still plenty of time left for the FCIC to carry on. +I sorted out the scraps of Evelyn's SPRINT bill and re-assembled them with +fresh Scotch tape. Here was her ten-digit FONCARD number. Didn't seem +to have the ID number necessary to cause real fraud trouble. + +I did, however, have Evelyn's home phone number. And the phone numbers +for a whole crowd of Evelyn's long-distance friends and acquaintances. +In San Diego, Folsom, Redondo, Las Vegas, La Jolla, Topeka, and Northampton +Massachusetts. Even somebody in Australia! + +I examined other documents. Here was a bank statement. It was Evelyn's +IRA account down at a bank in San Mateo California (total balance $1877.20). +Here was a charge-card bill for $382.64. She was paying it off bit by bit. + +Driven by motives that were completely unethical and prurient, +I now examined the handwritten notes. They had been torn fairly +thoroughly, so much so that it took me almost an entire five minutes +to reassemble them. + +They were drafts of a love letter. They had been written on +the lined stationery of Evelyn's employer, a biomedical company. +Probably written at work when she should have been doing something else. + +"Dear Bob," (not his real name) "I guess in everyone's life there comes +a time when hard decisions have to be made, and this is a difficult one +for me--very upsetting. Since you haven't called me, and I don't understand +why, I can only surmise it's because you don't want to. I thought I would +have heard from you Friday. I did have a few unusual problems with my phone +and possibly you tried, I hope so. + +"Robert, you asked me to `let go'. . . ." + +The first note ended. UNUSUAL PROBLEMS WITH HER PHONE? +I looked swiftly at the next note. + +"Bob, not hearing from you for the whole weekend has left me very perplexed. . . ." + +Next draft. + +"Dear Bob, there is so much I don't understand right now, and I wish I did. +I wish I could talk to you, but for some unknown reason you have elected not +to call--this is so difficult for me to understand. . . ." + +She tried again. + +"Bob, Since I have always held you in such high esteem, I had every hope that +we could remain good friends, but now one essential ingredient is missing-- +respect. Your ability to discard people when their purpose is served is +appalling to me. The kindest thing you could do for me now is to leave me +alone. You are no longer welcome in my heart or home. . . ." + +Try again. + +"Bob, I wrote a very factual note to you to say how much respect I had lost +for you, by the way you treat people, me in particular, so uncaring and cold. +The kindest thing you can do for me is to leave me alone entirely, +as you are no longer welcome in my heart or home. I would appreciate it +if you could retire your debt to me as soon as possible--I wish no link +to you in any way. Sincerely, Evelyn." + +Good heavens, I thought, the bastard actually owes her money! +I turned to the next page. + +"Bob: very simple. GOODBYE! No more mind games--no more fascination-- +no more coldness--no more respect for you! It's over--Finis. Evie" + +There were two versions of the final brushoff letter, but they read about +the same. Maybe she hadn't sent it. The final item in my illicit and +shameful booty was an envelope addressed to "Bob" at his home address, +but it had no stamp on it and it hadn't been mailed. + +Maybe she'd just been blowing off steam because her rascal boyfriend +had neglected to call her one weekend. Big deal. Maybe they'd kissed +and made up, maybe she and Bob were down at Pop's Chocolate Shop now, +sharing a malted. Sure. + +Easy to find out. All I had to do was call Evelyn up. With a half-clever +story and enough brass-plated gall I could probably trick the truth out of her. +Phone-phreaks and hackers deceive people over the phone all the time. +It's called "social engineering." Social engineering is a very common practice +in the underground, and almost magically effective. Human beings are almost +always the weakest link in computer security. The simplest way to learn +Things You Are Not Meant To Know is simply to call up and exploit the +knowledgeable people. With social engineering, you use the bits of specialized +knowledge you already have as a key, to manipulate people into believing +that you are legitimate. You can then coax, flatter, or frighten them into +revealing almost anything you want to know. Deceiving people (especially +over the phone) is easy and fun. Exploiting their gullibility is very +gratifying; it makes you feel very superior to them. + +If I'd been a malicious hacker on a trashing raid, I would now have Evelyn +very much in my power. Given all this inside data, it wouldn't take much +effort at all to invent a convincing lie. If I were ruthless enough, +and jaded enough, and clever enough, this momentary indiscretion of hers-- +maybe committed in tears, who knows--could cause her a whole world of +confusion and grief. + +I didn't even have to have a MALICIOUS motive. Maybe I'd be "on her side," +and call up Bob instead, and anonymously threaten to break both his kneecaps +if he didn't take Evelyn out for a steak dinner pronto. It was still +profoundly NONE OF MY BUSINESS. To have gotten this knowledge at all +was a sordid act and to use it would be to inflict a sordid injury. + +To do all these awful things would require exactly zero high-tech expertise. +All it would take was the willingness to do it and a certain amount +of bent imagination. + +I went back downstairs. The hard-working FCIC, who had labored forty-five +minutes over their schedule, were through for the day, and adjourned to the +hotel bar. We all had a beer. + +I had a chat with a guy about "Isis," or rather IACIS, +the International Association of Computer Investigation Specialists. +They're into "computer forensics," the techniques of picking computer- +systems apart without destroying vital evidence. IACIS, currently run +out of Oregon, is comprised of investigators in the U.S., Canada, Taiwan +and Ireland. "Taiwan and Ireland?" I said. Are TAIWAN and IRELAND +really in the forefront of this stuff? Well not exactly, my informant +admitted. They just happen to have been the first ones to have caught +on by word of mouth. Still, the international angle counts, because this +is obviously an international problem. Phone-lines go everywhere. + +There was a Mountie here from the Royal Canadian Mounted Police. +He seemed to be having quite a good time. Nobody had flung this +Canadian out because he might pose a foreign security risk. +These are cyberspace cops. They still worry a lot about "jurisdictions," +but mere geography is the least of their troubles. + +NASA had failed to show. NASA suffers a lot from computer intrusions, +in particular from Australian raiders and a well-trumpeted Chaos +Computer Club case, and in 1990 there was a brief press flurry +when it was revealed that one of NASA's Houston branch-exchanges +had been systematically ripped off by a gang of phone-phreaks. +But the NASA guys had had their funding cut. They were stripping everything. + +Air Force OSI, its Office of Special Investigations, is the ONLY federal +entity dedicated full-time to computer security. They'd been expected +to show up in force, but some of them had cancelled--a Pentagon budget pinch. + +As the empties piled up, the guys began joshing around and telling war-stories. +"These are cops," Thackeray said tolerantly. "If they're not talking shop +they talk about women and beer." + +I heard the story about the guy who, asked for "a copy" of a computer disk, +PHOTOCOPIED THE LABEL ON IT. He put the floppy disk onto the glass plate +of a photocopier. The blast of static when the copier worked completely +erased all the real information on the disk. + +Some other poor souls threw a whole bag of confiscated diskettes +into the squad-car trunk next to the police radio. The powerful radio +signal blasted them, too. + +We heard a bit about Dave Geneson, the first computer prosecutor, +a mainframe-runner in Dade County, turned lawyer. Dave Geneson +was one guy who had hit the ground running, a signal virtue +in making the transition to computer-crime. It was generally +agreed that it was easier to learn the world of computers first, +then police or prosecutorial work. You could take certain computer +people and train 'em to successful police work--but of course they +had to have the COP MENTALITY. They had to have street smarts. +Patience. Persistence. And discretion. You've got to make sure +they're not hot-shots, show-offs, "cowboys." + +Most of the folks in the bar had backgrounds in military intelligence, +or drugs, or homicide. It was rudely opined that "military intelligence" +was a contradiction in terms, while even the grisly world of homicide +was considered cleaner than drug enforcement. One guy had been 'way +undercover doing dope-work in Europe for four years straight. +"I'm almost recovered now," he said deadpan, with the acid black humor +that is pure cop. "Hey, now I can say FUCKER without putting MOTHER +in front of it." + +"In the cop world," another guy said earnestly, "everything is good and bad, +black and white. In the computer world everything is gray." + +One guy--a founder of the FCIC, who'd been with the group +since it was just the Colluquy--described his own introduction +to the field. He'd been a Washington DC homicide guy called in +on a "hacker" case. From the word "hacker," he naturally assumed +he was on the trail of a knife-wielding marauder, and went to the +computer center expecting blood and a body. When he finally figured +out what was happening there (after loudly demanding, in vain, +that the programmers "speak English"), he called headquarters +and told them he was clueless about computers. They told him nobody +else knew diddly either, and to get the hell back to work. + +So, he said, he had proceeded by comparisons. By analogy. By metaphor. +"Somebody broke in to your computer, huh?" Breaking and entering; +I can understand that. How'd he get in? "Over the phone-lines." +Harassing phone-calls, I can understand that! What we need here +is a tap and a trace! + +It worked. It was better than nothing. And it worked a lot faster +when he got hold of another cop who'd done something similar. +And then the two of them got another, and another, and pretty soon +the Colluquy was a happening thing. It helped a lot that everybody +seemed to know Carlton Fitzpatrick, the data-processing trainer in Glynco. + +The ice broke big-time in Memphis in '86. The Colluquy had attracted +a bunch of new guys--Secret Service, FBI, military, other feds, heavy guys. +Nobody wanted to tell anybody anything. They suspected that if word got back +to the home office they'd all be fired. They passed an uncomfortably +guarded afternoon. + +The formalities got them nowhere. But after the formal session was over, +the organizers brought in a case of beer. As soon as the participants +knocked it off with the bureaucratic ranks and turf-fighting, everything +changed. "I bared my soul," one veteran reminisced proudly. By nightfall +they were building pyramids of empty beer-cans and doing everything +but composing a team fight song. + +FCIC were not the only computer-crime people around. There was DATTA +(District Attorneys' Technology Theft Association), though they mostly +specialized in chip theft, intellectual property, and black-market cases. +There was HTCIA (High Tech Computer Investigators Association), +also out in Silicon Valley, a year older than FCIC and featuring +brilliant people like Donald Ingraham. There was LEETAC +(Law Enforcement Electronic Technology Assistance Committee) +in Florida, and computer-crime units in Illinois and Maryland +and Texas and Ohio and Colorado and Pennsylvania. But these were +local groups. FCIC were the first to really network nationally +and on a federal level. + +FCIC people live on the phone lines. Not on bulletin board systems-- +they know very well what boards are, and they know that boards aren't secure. +Everyone in the FCIC has a voice-phone bill like you wouldn't believe. +FCIC people have been tight with the telco people for a long time. +Telephone cyberspace is their native habitat. + +FCIC has three basic sub-tribes: the trainers, the security people, +and the investigators. That's why it's called an "Investigations +Committee" with no mention of the term "computer-crime"--the dreaded +"C-word." FCIC, officially, is "an association of agencies rather +than individuals;" unofficially, this field is small enough that +the influence of individuals and individual expertise is paramount. +Attendance is by invitation only, and most everyone in FCIC considers +himself a prophet without honor in his own house. + +Again and again I heard this, with different terms but identical +sentiments. "I'd been sitting in the wilderness talking to myself." +"I was totally isolated." "I was desperate." "FCIC is the best +thing there is about computer crime in America." "FCIC is what +really works." "This is where you hear real people telling you +what's really happening out there, not just lawyers picking nits." +"We taught each other everything we knew." + +The sincerity of these statements convinces me that this is true. +FCIC is the real thing and it is invaluable. It's also very sharply +at odds with the rest of the traditions and power structure +in American law enforcement. There probably hasn't been anything +around as loose and go-getting as the FCIC since the start of the +U.S. Secret Service in the 1860s. FCIC people are living like +twenty-first-century people in a twentieth-century environment, +and while there's a great deal to be said for that, there's also +a great deal to be said against it, and those against it happen +to control the budgets. + +I listened to two FCIC guys from Jersey compare life histories. +One of them had been a biker in a fairly heavy-duty gang in the 1960s. +"Oh, did you know so-and-so?" said the other guy from Jersey. +"Big guy, heavyset?" + +"Yeah, I knew him." + +"Yeah, he was one of ours. He was our plant in the gang." + +"Really? Wow! Yeah, I knew him. Helluva guy." + +Thackeray reminisced at length about being tear-gassed blind +in the November 1969 antiwar protests in Washington Circle, +covering them for her college paper. "Oh yeah, I was there," +said another cop. "Glad to hear that tear gas hit somethin'. +Haw haw haw." He'd been so blind himself, he confessed, +that later that day he'd arrested a small tree. + +FCIC are an odd group, sifted out by coincidence and necessity, +and turned into a new kind of cop. There are a lot of specialized +cops in the world--your bunco guys, your drug guys, your tax guys, +but the only group that matches FCIC for sheer isolation are probably +the child-pornography people. Because they both deal with conspirators +who are desperate to exchange forbidden data and also desperate to hide; +and because nobody else in law enforcement even wants to hear about it. + +FCIC people tend to change jobs a lot. They tend not to get the equipment +and training they want and need. And they tend to get sued quite often. + +As the night wore on and a band set up in the bar, the talk grew darker. +Nothing ever gets done in government, someone opined, until there's +a DISASTER. Computing disasters are awful, but there's no denying +that they greatly help the credibility of FCIC people. The Internet Worm, +for instance. "For years we'd been warning about that--but it's nothing +compared to what's coming." They expect horrors, these people. +They know that nothing will really get done until there is a horror. + +# + +Next day we heard an extensive briefing from a guy who'd been a computer cop, +gotten into hot water with an Arizona city council, and now installed +computer networks for a living (at a considerable rise in pay). +He talked about pulling fiber-optic networks apart. + +Even a single computer, with enough peripherals, is a literal +"network"--a bunch of machines all cabled together, generally +with a complexity that puts stereo units to shame. FCIC people +invent and publicize methods of seizing computers and maintaining +their evidence. Simple things, sometimes, but vital rules of thumb +for street cops, who nowadays often stumble across a busy computer +in the midst of a drug investigation or a white-collar bust. +For instance: Photograph the system before you touch it. +Label the ends of all the cables before you detach anything. +"Park" the heads on the disk drives before you move them. +Get the diskettes. Don't put the diskettes in magnetic fields. +Don't write on diskettes with ballpoint pens. Get the manuals. +Get the printouts. Get the handwritten notes. Copy data before +you look at it, and then examine the copy instead of the original. + +Now our lecturer distributed copied diagrams of a typical LAN +or "Local Area Network", which happened to be out of Connecticut. +ONE HUNDRED AND FIFTY-NINE desktop computers, each with its own +peripherals. Three "file servers." Five "star couplers" +each with thirty-two ports. One sixteen-port coupler +off in the corner office. All these machines talking to each other, +distributing electronic mail, distributing software, distributing, +quite possibly, criminal evidence. All linked by high-capacity +fiber-optic cable. A bad guy--cops talk a about "bad guys" +--might be lurking on PC #47 lot or #123 and distributing +his ill doings onto some dupe's "personal" machine in +another office--or another floor--or, quite possibly, +two or three miles away! Or, conceivably, the evidence might +be "data-striped"--split up into meaningless slivers stored, +one by one, on a whole crowd of different disk drives. + +The lecturer challenged us for solutions. I for one was utterly clueless. +As far as I could figure, the Cossacks were at the gate; there were probably +more disks in this single building than were seized during the entirety +of Operation Sundevil. + +"Inside informant," somebody said. Right. There's always the human angle, +something easy to forget when contemplating the arcane recesses of high +technology. Cops are skilled at getting people to talk, and computer people, +given a chair and some sustained attention, will talk about their computers +till their throats go raw. There's a case on record of a single question-- +"How'd you do it?"--eliciting a forty-five-minute videotaped confession +from a computer criminal who not only completely incriminated himself +but drew helpful diagrams. + +Computer people talk. Hackers BRAG. Phone-phreaks +talk PATHOLOGICALLY--why else are they stealing phone-codes, +if not to natter for ten hours straight to their friends +on an opposite seaboard? Computer-literate people do +in fact possess an arsenal of nifty gadgets and techniques +that would allow them to conceal all kinds of exotic skullduggery, +and if they could only SHUT UP about it, they could probably +get away with all manner of amazing information-crimes. +But that's just not how it works--or at least, +that's not how it's worked SO FAR. + +Most every phone-phreak ever busted has swiftly implicated his mentors, +his disciples, and his friends. Most every white-collar computer-criminal, +smugly convinced that his clever scheme is bulletproof, swiftly learns +otherwise when, for the first time in his life, an actual no-kidding +policeman leans over, grabs the front of his shirt, looks him right +in the eye and says: "All right, ASSHOLE--you and me are going downtown!" +All the hardware in the world will not insulate your nerves from +these actual real-life sensations of terror and guilt. + +Cops know ways to get from point A to point Z without thumbing +through every letter in some smart-ass bad-guy's alphabet. +Cops know how to cut to the chase. Cops know a lot of things +other people don't know. + +Hackers know a lot of things other people don't know, too. +Hackers know, for instance, how to sneak into your computer +through the phone-lines. But cops can show up RIGHT ON YOUR DOORSTEP +and carry off YOU and your computer in separate steel boxes. +A cop interested in hackers can grab them and grill them. +A hacker interested in cops has to depend on hearsay, +underground legends, and what cops are willing to publicly reveal. +And the Secret Service didn't get named "the SECRET Service" +because they blab a lot. + +Some people, our lecturer informed us, were under the mistaken +impression that it was "impossible" to tap a fiber-optic line. +Well, he announced, he and his son had just whipped up a +fiber-optic tap in his workshop at home. He passed it around +the audience, along with a circuit-covered LAN plug-in card +so we'd all recognize one if we saw it on a case. We all had a look. + +The tap was a classic "Goofy Prototype"--a thumb-length rounded +metal cylinder with a pair of plastic brackets on it. +From one end dangled three thin black cables, each of which ended +in a tiny black plastic cap. When you plucked the safety-cap +off the end of a cable, you could see the glass fiber-- +no thicker than a pinhole. + +Our lecturer informed us that the metal cylinder was a +"wavelength division multiplexer." Apparently, what one did +was to cut the fiber-optic cable, insert two of the legs into +the cut to complete the network again, and then read any passing data +on the line by hooking up the third leg to some kind of monitor. +Sounded simple enough. I wondered why nobody had thought of it before. +I also wondered whether this guy's son back at the workshop had any +teenage friends. + +We had a break. The guy sitting next to me was wearing a giveaway +baseball cap advertising the Uzi submachine gun. We had a desultory chat +about the merits of Uzis. Long a favorite of the Secret Service, +it seems Uzis went out of fashion with the advent of the Persian Gulf War, +our Arab allies taking some offense at Americans toting Israeli weapons. +Besides, I was informed by another expert, Uzis jam. The equivalent weapon +of choice today is the Heckler & Koch, manufactured in Germany. + +The guy with the Uzi cap was a forensic photographer. He also did a lot +of photographic surveillance work in computer crime cases. He used to, +that is, until the firings in Phoenix. He was now a private investigator and, +with his wife, ran a photography salon specializing in weddings and portrait +photos. At--one must repeat--a considerable rise in income. + +He was still FCIC. If you were FCIC, and you needed to talk +to an expert about forensic photography, well, there he was, +willing and able. If he hadn't shown up, people would have missed him. + +Our lecturer had raised the point that preliminary investigation +of a computer system is vital before any seizure is undertaken. +It's vital to understand how many machines are in there, what kinds +there are, what kind of operating system they use, how many people +use them, where the actual data itself is stored. To simply barge into +an office demanding "all the computers" is a recipe for swift disaster. + +This entails some discreet inquiries beforehand. In fact, what it +entails is basically undercover work. An intelligence operation. +SPYING, not to put too fine a point on it. + +In a chat after the lecture, I asked an attendee whether "trashing" might work. + +I received a swift briefing on the theory and practice of "trash covers." +Police "trash covers," like "mail covers" or like wiretaps, require the +agreement of a judge. This obtained, the "trashing" work of cops is just +like that of hackers, only more so and much better organized. So much so, +I was informed, that mobsters in Phoenix make extensive use of locked +garbage cans picked up by a specialty high-security trash company. + +In one case, a tiger team of Arizona cops had trashed a local residence +for four months. Every week they showed up on the municipal garbage truck, +disguised as garbagemen, and carried the contents of the suspect cans off +to a shade tree, where they combed through the garbage--a messy task, +especially considering that one of the occupants was undergoing +kidney dialysis. All useful documents were cleaned, dried and examined. +A discarded typewriter-ribbon was an especially valuable source of data, +as its long one-strike ribbon of film contained the contents of every +letter mailed out of the house. The letters were neatly retyped by +a police secretary equipped with a large desk-mounted magnifying glass. + +There is something weirdly disquieting about the whole subject of +"trashing"-- an unsuspected and indeed rather disgusting mode of +deep personal vulnerability. Things that we pass by every day, +that we take utterly for granted, can be exploited with so little work. +Once discovered, the knowledge of these vulnerabilities tend to spread. + +Take the lowly subject of MANHOLE COVERS. The humble manhole cover +reproduces many of the dilemmas of computer-security in miniature. +Manhole covers are, of course, technological artifacts, access-points +to our buried urban infrastructure. To the vast majority of us, +manhole covers are invisible. They are also vulnerable. For many years now, +the Secret Service has made a point of caulking manhole covers along all routes +of the Presidential motorcade. This is, of course, to deter terrorists from +leaping out of underground ambush or, more likely, planting remote-control +car-smashing bombs beneath the street. + +Lately, manhole covers have seen more and more criminal exploitation, +especially in New York City. Recently, a telco in New York City +discovered that a cable television service had been sneaking into +telco manholes and installing cable service alongside the phone-lines-- +WITHOUT PAYING ROYALTIES. New York companies have also suffered a +general plague of (a) underground copper cable theft; (b) dumping of garbage, +including toxic waste, and (c) hasty dumping of murder victims. + +Industry complaints reached the ears of an innovative New England +industrial-security company, and the result was a new product known +as "the Intimidator," a thick titanium-steel bolt with a precisely machined +head that requires a special device to unscrew. All these "keys" have registered +serial numbers kept on file with the manufacturer. There are now some +thousands of these "Intimidator" bolts being sunk into American pavements +wherever our President passes, like some macabre parody of strewn roses. +They are also spreading as fast as steel dandelions around US military bases +and many centers of private industry. + +Quite likely it has never occurred to you to peer under a manhole cover, +perhaps climb down and walk around down there with a flashlight, just to see +what it's like. Formally speaking, this might be trespassing, but if you +didn't hurt anything, and didn't make an absolute habit of it, nobody would +really care. The freedom to sneak under manholes was likely a freedom +you never intended to exercise. + +You now are rather less likely to have that freedom at all. +You may never even have missed it until you read about it here, +but if you're in New York City it's gone, and elsewhere it's likely going. +This is one of the things that crime, and the reaction to +crime, does to us. + +The tenor of the meeting now changed as the Electronic Frontier Foundation +arrived. The EFF, whose personnel and history will be examined in detail +in the next chapter, are a pioneering civil liberties group who arose in +direct response to the Hacker Crackdown of 1990. + +Now Mitchell Kapor, the Foundation's president, and Michael Godwin, +its chief attorney, were confronting federal law enforcement MANO A MANO +for the first time ever. Ever alert to the manifold uses of publicity, +Mitch Kapor and Mike Godwin had brought their own journalist in tow: +Robert Draper, from Austin, whose recent well-received book about +ROLLING STONE magazine was still on the stands. Draper was on assignment +for TEXAS MONTHLY. + +The Steve Jackson/EFF civil lawsuit against the Chicago Computer Fraud +and Abuse Task Force was a matter of considerable regional interest in Texas. +There were now two Austinite journalists here on the case. In fact, +counting Godwin (a former Austinite and former journalist) there were +three of us. Lunch was like Old Home Week. + +Later, I took Draper up to my hotel room. We had a long frank talk +about the case, networking earnestly like a miniature freelance-journo +version of the FCIC: privately confessing the numerous blunders +of journalists covering the story, and trying hard to figure out +who was who and what the hell was really going on out there. +I showed Draper everything I had dug out of the Hilton trashcan. +We pondered the ethics of "trashing" for a while, and agreed +that they were dismal. We also agreed that finding a SPRINT +bill on your first time out was a heck of a coincidence. + +First I'd "trashed"--and now, mere hours later, I'd bragged to someone else. +Having entered the lifestyle of hackerdom, I was now, unsurprisingly, +following its logic. Having discovered something remarkable through +a surreptitious action, I of course HAD to "brag," and to drag the passing +Draper into my iniquities. I felt I needed a witness. Otherwise nobody +would have believed what I'd discovered. . . . + +Back at the meeting, Thackeray cordially, if rather tentatively, +introduced Kapor and Godwin to her colleagues. Papers were distributed. +Kapor took center stage. The brilliant Bostonian high-tech entrepreneur, +normally the hawk in his own administration and quite an effective +public speaker, seemed visibly nervous, and frankly admitted as much. +He began by saying he consided computer-intrusion to be morally wrong, +and that the EFF was not a "hacker defense fund," despite what had appeared +in print. Kapor chatted a bit about the basic motivations of his group, +emphasizing their good faith and willingness to listen and seek common ground +with law enforcement--when, er, possible. + +Then, at Godwin's urging, Kapor suddenly remarked that EFF's own Internet +machine had been "hacked" recently, and that EFF did not consider +this incident amusing. + +After this surprising confession, things began to loosen up +quite rapidly. Soon Kapor was fielding questions, parrying objections, +challenging definitions, and juggling paradigms with something akin +to his usual gusto. + +Kapor seemed to score quite an effect with his shrewd and skeptical analysis +of the merits of telco "Caller-ID" services. (On this topic, FCIC and EFF +have never been at loggerheads, and have no particular established earthworks +to defend.) Caller-ID has generally been promoted as a privacy service +for consumers, a presentation Kapor described as a "smokescreen," +the real point of Caller-ID being to ALLOW CORPORATE CUSTOMERS TO BUILD +EXTENSIVE COMMERCIAL DATABASES ON EVERYBODY WHO PHONES OR FAXES THEM. +Clearly, few people in the room had considered this possibility, +except perhaps for two late-arrivals from US WEST RBOC security, +who chuckled nervously. + +Mike Godwin then made an extensive presentation on +"Civil Liberties Implications of Computer Searches and Seizures." +Now, at last, we were getting to the real nitty-gritty here, +real political horse-trading. The audience listened with close +attention, angry mutters rising occasionally: "He's trying to +teach us our jobs!" "We've been thinking about this for years! +We think about these issues every day!" "If I didn't seize the works, +I'd be sued by the guy's victims!" "I'm violating the law if I leave +ten thousand disks full of illegal PIRATED SOFTWARE and STOLEN CODES!" +"It's our job to make sure people don't trash the Constitution-- +we're the DEFENDERS of the Constitution!" "We seize stuff when +we know it will be forfeited anyway as restitution for the victim!" + +"If it's forfeitable, then don't get a search warrant, get a +forfeiture warrant," Godwin suggested coolly. He further remarked +that most suspects in computer crime don't WANT to see their computers +vanish out the door, headed God knew where, for who knows how long. +They might not mind a search, even an extensive search, but they want +their machines searched on-site. + +"Are they gonna feed us?" somebody asked sourly. + +"How about if you take copies of the data?" Godwin parried. + +"That'll never stand up in court." + +"Okay, you make copies, give THEM the copies, and take the originals." + +Hmmm. + +Godwin championed bulletin-board systems as repositories of First Amendment +protected free speech. He complained that federal computer-crime training +manuals gave boards a bad press, suggesting that they are hotbeds of crime +haunted by pedophiles and crooks, whereas the vast majority of the nation's +thousands of boards are completely innocuous, and nowhere near so +romantically suspicious. + +People who run boards violently resent it when their systems are seized, +and their dozens (or hundreds) of users look on in abject horror. +Their rights of free expression are cut short. Their right to associate +with other people is infringed. And their privacy is violated as their +private electronic mail becomes police property. + +Not a soul spoke up to defend the practice of seizing boards. +The issue passed in chastened silence. Legal principles aside-- +(and those principles cannot be settled without laws passed or +court precedents)--seizing bulletin boards has become public-relations +poison for American computer police. + +And anyway, it's not entirely necessary. If you're a cop, you can get 'most +everything you need from a pirate board, just by using an inside informant. +Plenty of vigilantes--well, CONCERNED CITIZENS--will inform police the moment +they see a pirate board hit their area (and will tell the police all about it, +in such technical detail, actually, that you kinda wish they'd shut up). +They will happily supply police with extensive downloads or printouts. +It's IMPOSSIBLE to keep this fluid electronic information out of the +hands of police. + +Some people in the electronic community become enraged at the prospect +of cops "monitoring" bulletin boards. This does have touchy aspects, +as Secret Service people in particular examine bulletin boards with +some regularity. But to expect electronic police to be deaf dumb +and blind in regard to this particular medium rather flies in the face +of common sense. Police watch television, listen to radio, read newspapers +and magazines; why should the new medium of boards be different? +Cops can exercise the same access to electronic information +as everybody else. As we have seen, quite a few computer +police maintain THEIR OWN bulletin boards, including anti-hacker +"sting" boards, which have generally proven quite effective. + +As a final clincher, their Mountie friends in Canada (and colleagues +in Ireland and Taiwan) don't have First Amendment or American +constitutional restrictions, but they do have phone lines, +and can call any bulletin board in America whenever they please. +The same technological determinants that play into the hands of hackers, +phone phreaks and software pirates can play into the hands of police. +"Technological determinants" don't have ANY human allegiances. +They're not black or white, or Establishment or Underground, +or pro-or-anti anything. + +Godwin complained at length about what he called "the Clever Hobbyist +hypothesis" --the assumption that the "hacker" you're busting is clearly +a technical genius, and must therefore by searched with extreme thoroughness. +So: from the law's point of view, why risk missing anything? Take the works. +Take the guy's computer. Take his books. Take his notebooks. +Take the electronic drafts of his love letters. Take his Walkman. +Take his wife's computer. Take his dad's computer. Take his kid +sister's computer. Take his employer's computer. Take his compact disks-- +they MIGHT be CD-ROM disks, cunningly disguised as pop music. +Take his laser printer--he might have hidden something vital in the +printer's 5meg of memory. Take his software manuals and hardware +documentation. Take his science-fiction novels and his simulation- +gaming books. Take his Nintendo Game-Boy and his Pac-Man arcade game. +Take his answering machine, take his telephone out of the wall. +Take anything remotely suspicious. + +Godwin pointed out that most "hackers" are not, in fact, clever +genius hobbyists. Quite a few are crooks and grifters who don't +have much in the way of technical sophistication; just some rule-of-thumb +rip-off techniques. The same goes for most fifteen-year-olds who've +downloaded a code-scanning program from a pirate board. There's no +real need to seize everything in sight. It doesn't require an entire +computer system and ten thousand disks to prove a case in court. + +What if the computer is the instrumentality of a crime? someone demanded. + +Godwin admitted quietly that the doctrine of seizing the instrumentality +of a crime was pretty well established in the American legal system. + +The meeting broke up. Godwin and Kapor had to leave. Kapor was testifying +next morning before the Massachusetts Department Of Public Utility, +about ISDN narrowband wide-area networking. + +As soon as they were gone, Thackeray seemed elated. +She had taken a great risk with this. Her colleagues had not, +in fact, torn Kapor and Godwin's heads off. She was very proud of them, +and told them so. + +"Did you hear what Godwin said about INSTRUMENTALITY OF A CRIME?" +she exulted, to nobody in particular. "Wow, that means +MITCH ISN'T GOING TO SUE ME." + +# + +America's computer police are an interesting group. +As a social phenomenon they are far more interesting, +and far more important, than teenage phone phreaks +and computer hackers. First, they're older and wiser; +not dizzy hobbyists with leaky morals, but seasoned adult +professionals with all the responsibilities of public service. +And, unlike hackers, they possess not merely TECHNICAL +power alone, but heavy-duty legal and social authority. + +And, very interestingly, they are just as much at +sea in cyberspace as everyone else. They are not +happy about this. Police are authoritarian by nature, +and prefer to obey rules and precedents. (Even those police +who secretly enjoy a fast ride in rough territory will soberly +disclaim any "cowboy" attitude.) But in cyberspace there ARE +no rules and precedents. They are groundbreaking pioneers, +Cyberspace Rangers, whether they like it or not. + +In my opinion, any teenager enthralled by computers, +fascinated by the ins and outs of computer security, +and attracted by the lure of specialized forms of knowledge and power, +would do well to forget all about "hacking" and set his (or her) +sights on becoming a fed. Feds can trump hackers at almost every +single thing hackers do, including gathering intelligence, +undercover disguise, trashing, phone-tapping, building dossiers, +networking, and infiltrating computer systems--CRIMINAL computer systems. +Secret Service agents know more about phreaking, coding and carding +than most phreaks can find out in years, and when it comes to viruses, +break-ins, software bombs and trojan horses, Feds have direct access to red-hot +confidential information that is only vague rumor in the underground. + +And if it's an impressive public rep you're after, there are few people +in the world who can be so chillingly impressive as a well-trained, +well-armed United States Secret Service agent. + +Of course, a few personal sacrifices are necessary in order to obtain +that power and knowledge. First, you'll have the galling discipline +of belonging to a large organization; but the world of computer crime +is still so small, and so amazingly fast-moving, that it will remain +spectacularly fluid for years to come. The second sacrifice is that +you'll have to give up ripping people off. This is not a great loss. +Abstaining from the use of illegal drugs, also necessary, will be a boon +to your health. + +A career in computer security is not a bad choice for a young man +or woman today. The field will almost certainly expand drastically +in years to come. If you are a teenager today, by the time you +become a professional, the pioneers you have read about in this book +will be the grand old men and women of the field, swamped by their many +disciples and successors. Of course, some of them, like William P. Wood +of the 1865 Secret Service, may well be mangled in the whirring machinery +of legal controversy; but by the time you enter the computer-crime field, +it may have stabilized somewhat, while remaining entertainingly challenging. + +But you can't just have a badge. You have to win it. First, there's the +federal law enforcement training. And it's hard--it's a challenge. +A real challenge--not for wimps and rodents. + +Every Secret Service agent must complete gruelling courses at the +Federal Law Enforcement Training Center. (In fact, Secret Service +agents are periodically re-trained during their entire careers.) + +In order to get a glimpse of what this might be like, +I myself travelled to FLETC. + +# + +The Federal Law Enforcement Training Center is a 1500-acre facility +on Georgia's Atlantic coast. It's a milieu of marshgrass, seabirds, +damp, clinging sea-breezes, palmettos, mosquitos, and bats. +Until 1974, it was a Navy Air Base, and still features a working runway, +and some WWII vintage blockhouses and officers' quarters. +The Center has since benefitted by a forty-million-dollar retrofit, +but there's still enough forest and swamp on the facility for the +Border Patrol to put in tracking practice. + +As a town, "Glynco" scarcely exists. The nearest real town is Brunswick, +a few miles down Highway 17, where I stayed at the aptly named Marshview +Holiday Inn. I had Sunday dinner at a seafood restaurant called "Jinright's," +where I feasted on deep-fried alligator tail. This local favorite was +a heaped basket of bite-sized chunks of white, tender, almost fluffy +reptile meat, steaming in a peppered batter crust. Alligator makes +a culinary experience that's hard to forget, especially when liberally +basted with homemade cocktail sauce from a Jinright squeeze-bottle. + +The crowded clientele were tourists, fishermen, local black folks +in their Sunday best, and white Georgian locals who all seemed +to bear an uncanny resemblance to Georgia humorist Lewis Grizzard. + +The 2,400 students from 75 federal agencies who make up the FLETC +population scarcely seem to make a dent in the low-key local scene. +The students look like tourists, and the teachers seem to have taken +on much of the relaxed air of the Deep South. My host was Mr. Carlton +Fitzpatrick, the Program Coordinator of the Financial Fraud Institute. +Carlton Fitzpatrick is a mustached, sinewy, well-tanned Alabama native +somewhere near his late forties, with a fondness for chewing tobacco, +powerful computers, and salty, down-home homilies. We'd met before, +at FCIC in Arizona. + +The Financial Fraud Institute is one of the nine divisions at FLETC. +Besides Financial Fraud, there's Driver & Marine, Firearms, +and Physical Training. These are specialized pursuits. +There are also five general training divisions: Basic Training, +Operations, Enforcement Techniques, Legal Division, and Behavioral Science. + +Somewhere in this curriculum is everything necessary to turn green college +graduates into federal agents. First they're given ID cards. Then they get +the rather miserable-looking blue coveralls known as "smurf suits." +The trainees are assigned a barracks and a cafeteria, and immediately +set on FLETC's bone-grinding physical training routine. Besides the +obligatory daily jogging--(the trainers run up danger flags beside +the track when the humidity rises high enough to threaten heat stroke)-- +here's the Nautilus machines, the martial arts, the survival skills. . . . + +The eighteen federal agencies who maintain on-site academies at FLETC +employ a wide variety of specialized law enforcement units, some of them +rather arcane. There's Border Patrol, IRS Criminal Investigation Division, +Park Service, Fish and Wildlife, Customs, Immigration, Secret Service and +the Treasury's uniformed subdivisions. . . . If you're a federal cop +and you don't work for the FBI, you train at FLETC. This includes people +as apparently obscure as the agents of the Railroad Retirement Board +Inspector General. Or the Tennessee Valley Authority Police, +who are in fact federal police officers, and can and do arrest criminals +on the federal property of the Tennessee Valley Authority. + +And then there are the computer-crime people. All sorts, all backgrounds. +Mr. Fitzpatrick is not jealous of his specialized knowledge. Cops all over, +in every branch of service, may feel a need to learn what he can teach. +Backgrounds don't matter much. Fitzpatrick himself was originally a +Border Patrol veteran, then became a Border Patrol instructor at FLETC. +His Spanish is still fluent--but he found himself strangely fascinated +when the first computers showed up at the Training Center. Fitzpatrick +did have a background in electrical engineering, and though he never +considered himself a computer hacker, he somehow found himself writing +useful little programs for this new and promising gizmo. + +He began looking into the general subject of computers and crime, +reading Donn Parker's books and articles, keeping an ear cocked +for war stories, useful insights from the field, the up-and-coming +people of the local computer-crime and high-technology units. . . . +Soon he got a reputation around FLETC as the resident "computer expert," +and that reputation alone brought him more exposure, more experience-- +until one day he looked around, and sure enough he WAS a federal +computer-crime expert. + +In fact, this unassuming, genial man may be THE federal computer-crime expert. +There are plenty of very good computer people, and plenty of very good +federal investigators, but the area where these worlds of expertise overlap +is very slim. And Carlton Fitzpatrick has been right at the center of that +since 1985, the first year of the Colluquy, a group which owes much to +his influence. + +He seems quite at home in his modest, acoustic-tiled office, +with its Ansel Adams-style Western photographic art, a gold-framed +Senior Instructor Certificate, and a towering bookcase crammed with +three-ring binders with ominous titles such as Datapro Reports on +Information Security and CFCA Telecom Security '90. + +The phone rings every ten minutes; colleagues show up at the door +to chat about new developments in locksmithing or to shake their heads +over the latest dismal developments in the BCCI global banking scandal. + +Carlton Fitzpatrick is a fount of computer-crime war-stories, +related in an acerbic drawl. He tells me the colorful tale +of a hacker caught in California some years back. He'd been +raiding systems, typing code without a detectable break, +for twenty, twenty-four, thirty-six hours straight. Not just +logged on--TYPING. Investigators were baffled. Nobody +could do that. Didn't he have to go to the bathroom? +Was it some kind of automatic keyboard-whacking device +that could actually type code? + +A raid on the suspect's home revealed a situation of astonishing squalor. +The hacker turned out to be a Pakistani computer-science student who had +flunked out of a California university. He'd gone completely underground +as an illegal electronic immigrant, and was selling stolen phone-service +to stay alive. The place was not merely messy and dirty, but in a state +of psychotic disorder. Powered by some weird mix of culture shock, +computer addiction, and amphetamines, the suspect had in fact been sitting +in front of his computer for a day and a half straight, with snacks and +drugs at hand on the edge of his desk and a chamber-pot under his chair. + +Word about stuff like this gets around in the hacker-tracker community. + +Carlton Fitzpatrick takes me for a guided tour by car around the +FLETC grounds. One of our first sights is the biggest indoor +firing range in the world. There are federal trainees in there, +Fitzpatrick assures me politely, blasting away with a wide variety +of automatic weapons: Uzis, Glocks, AK-47s. . . . He's willing to +take me inside. I tell him I'm sure that's really interesting, +but I'd rather see his computers. Carlton Fitzpatrick seems quite +surprised and pleased. I'm apparently the first journalist he's ever +seen who has turned down the shooting gallery in favor of microchips. + +Our next stop is a favorite with touring Congressmen: the three-mile +long FLETC driving range. Here trainees of the Driver & Marine Division +are taught high-speed pursuit skills, setting and breaking road-blocks, +diplomatic security driving for VIP limousines. . . . A favorite FLETC +pastime is to strap a passing Senator into the passenger seat beside a +Driver & Marine trainer, hit a hundred miles an hour, then take it right into +"the skid-pan," a section of greased track where two tons of Detroit iron +can whip and spin like a hockey puck. + +Cars don't fare well at FLETC. First they're rifled again and again +for search practice. Then they do 25,000 miles of high-speed +pursuit training; they get about seventy miles per set +of steel-belted radials. Then it's off to the skid pan, +where sometimes they roll and tumble headlong in the grease. +When they're sufficiently grease-stained, dented, and creaky, +they're sent to the roadblock unit, where they're battered without pity. +And finally then they're sacrificed to the Bureau of Alcohol, +Tobacco and Firearms, whose trainees learn the ins and outs +of car-bomb work by blowing them into smoking wreckage. + +There's a railroad box-car on the FLETC grounds, and a large +grounded boat, and a propless plane; all training-grounds for searches. +The plane sits forlornly on a patch of weedy tarmac next to an eerie +blockhouse known as the "ninja compound," where anti-terrorism specialists +practice hostage rescues. As I gaze on this creepy paragon of modern +low-intensity warfare, my nerves are jangled by a sudden staccato outburst +of automatic weapons fire, somewhere in the woods to my right. +"Nine-millimeter," Fitzpatrick judges calmly. + +Even the eldritch ninja compound pales somewhat compared +to the truly surreal area known as "the raid-houses." +This is a street lined on both sides with nondescript +concrete-block houses with flat pebbled roofs. +They were once officers' quarters. Now they are training grounds. +The first one to our left, Fitzpatrick tells me, has been specially +adapted for computer search-and-seizure practice. Inside it has been +wired for video from top to bottom, with eighteen pan-and-tilt +remotely controlled videocams mounted on walls and in corners. +Every movement of the trainee agent is recorded live by teachers, +for later taped analysis. Wasted movements, hesitations, possibly lethal +tactical mistakes--all are gone over in detail. + +Perhaps the weirdest single aspect of this building is its front door, +scarred and scuffed all along the bottom, from the repeated impact, +day after day, of federal shoe-leather. + +Down at the far end of the row of raid-houses some people are practicing +a murder. We drive by slowly as some very young and rather nervous-looking +federal trainees interview a heavyset bald man on the raid-house lawn. +Dealing with murder takes a lot of practice; first you have to learn +to control your own instinctive disgust and panic, then you have to learn +to control the reactions of a nerve-shredded crowd of civilians, +some of whom may have just lost a loved one, some of whom may be murderers-- +quite possibly both at once. + +A dummy plays the corpse. The roles of the bereaved, the morbidly curious, +and the homicidal are played, for pay, by local Georgians: waitresses, +musicians, most anybody who needs to moonlight and can learn a script. +These people, some of whom are FLETC regulars year after year, +must surely have one of the strangest jobs in the world. + +Something about the scene: "normal" people in a weird situation, +standing around talking in bright Georgia sunshine, unsuccessfully +pretending that something dreadful has gone on, while a dummy lies +inside on faked bloodstains. . . . While behind this weird masquerade, +like a nested set of Russian dolls, are grim future realities of real death, +real violence, real murders of real people, that these young agents +will really investigate, many times during their careers. . . . +Over and over. . . . Will those anticipated murders look like this, +feel like this--not as "real" as these amateur actors are trying to +make it seem, but both as "real," and as numbingly unreal, as watching +fake people standing around on a fake lawn? Something about this scene +unhinges me. It seems nightmarish to me, Kafkaesque. I simply don't +know how to take it; my head is turned around; I don't know whether to laugh, +cry, or just shudder. + +When the tour is over, Carlton Fitzpatrick and I talk about computers. +For the first time cyberspace seems like quite a comfortable place. +It seems very real to me suddenly, a place where I know what I'm talking about, +a place I'm used to. It's real. "Real." Whatever. + +Carlton Fitzpatrick is the only person I've met in cyberspace circles +who is happy with his present equipment. He's got a 5 Meg RAM PC with +a 112 meg hard disk; a 660 meg's on the way. He's got a Compaq 386 desktop, +and a Zenith 386 laptop with 120 meg. Down the hall is a NEC Multi-Sync 2A +with a CD-ROM drive and a 9600 baud modem with four com-lines. +There's a training minicomputer, and a 10-meg local mini just for the Center, +and a lab-full of student PC clones and half-a-dozen Macs or so. +There's a Data General MV 2500 with 8 meg on board and a 370 meg disk. + +Fitzpatrick plans to run a UNIX board on the Data General when he's +finished beta-testing the software for it, which he wrote himself. +It'll have E-mail features, massive files on all manner of computer-crime +and investigation procedures, and will follow the computer-security +specifics of the Department of Defense "Orange Book." He thinks +it will be the biggest BBS in the federal government. + +Will it have Phrack on it? I ask wryly. + +Sure, he tells me. Phrack, TAP, Computer Underground Digest, +all that stuff. With proper disclaimers, of course. + +I ask him if he plans to be the sysop. Running a system that size is very +time-consuming, and Fitzpatrick teaches two three-hour courses every day. + +No, he says seriously, FLETC has to get its money worth out of the instructors. +He thinks he can get a local volunteer to do it, a high-school student. + +He says a bit more, something I think about an Eagle Scout law-enforcement +liaison program, but my mind has rocketed off in disbelief. + +"You're going to put a TEENAGER in charge of a federal security BBS?" +I'm speechless. It hasn't escaped my notice that the FLETC Financial +Fraud Institute is the ULTIMATE hacker-trashing target; there is stuff in here, +stuff of such utter and consummate cool by every standard of the +digital underground. . . . + +I imagine the hackers of my acquaintance, fainting dead-away from +forbidden-knowledge greed-fits, at the mere prospect of cracking +the superultra top-secret computers used to train the Secret Service +in computer-crime. . . . + +"Uhm, Carlton," I babble, "I'm sure he's a really nice kid and all, +but that's a terrible temptation to set in front of somebody who's, +you know, into computers and just starting out. . . ." + +"Yeah," he says, "that did occur to me." For the first time I begin +to suspect that he's pulling my leg. + +He seems proudest when he shows me an ongoing project called JICC, +Joint Intelligence Control Council. It's based on the services provided +by EPIC, the El Paso Intelligence Center, which supplies data and intelligence +to the Drug Enforcement Administration, the Customs Service, the Coast Guard, +and the state police of the four southern border states. Certain EPIC files +can now be accessed by drug-enforcement police of Central America, +South America and the Caribbean, who can also trade information +among themselves. Using a telecom program called "White Hat," +written by two brothers named Lopez from the Dominican Republic, +police can now network internationally on inexpensive PCs. +Carlton Fitzpatrick is teaching a class of drug-war agents +from the Third World, and he's very proud of their progress. +Perhaps soon the sophisticated smuggling networks of the +Medellin Cartel will be matched by a sophisticated computer +network of the Medellin Cartel's sworn enemies. They'll track boats, +track contraband, track the international drug-lords who now leap over +borders with great ease, defeating the police through the clever use +of fragmented national jurisdictions. + +JICC and EPIC must remain beyond the scope of this book. +They seem to me to be very large topics fraught with complications +that I am not fit to judge. I do know, however, that the international, +computer-assisted networking of police, across national boundaries, +is something that Carlton Fitzpatrick considers very important, +a harbinger of a desirable future. I also know that networks +by their nature ignore physical boundaries. And I also know +that where you put communications you put a community, +and that when those communities become self-aware +they will fight to preserve themselves and to expand their influence. +I make no judgements whether this is good or bad. +It's just cyberspace; it's just the way things are. + +I asked Carlton Fitzpatrick what advice he would have for +a twenty-year-old who wanted to shine someday in the world +of electronic law enforcement. + +He told me that the number one rule was simply not to be +scared of computers. You don't need to be an obsessive +"computer weenie," but you mustn't be buffaloed just because +some machine looks fancy. The advantages computers give +smart crooks are matched by the advantages they give smart cops. +Cops in the future will have to enforce the law "with their heads, +not their holsters." Today you can make good cases without ever +leaving your office. In the future, cops who resist the computer +revolution will never get far beyond walking a beat. + +I asked Carlton Fitzpatrick if he had some single message for the public; +some single thing that he would most like the American public to know +about his work. + +He thought about it while. "Yes," he said finally. "TELL me the rules, +and I'll TEACH those rules!" He looked me straight in the eye. +"I do the best that I can." + + + +PART FOUR: THE CIVIL LIBERTARIANS + + +The story of the Hacker Crackdown, as we have followed it thus far, +has been technological, subcultural, criminal and legal. +The story of the Civil Libertarians, though it partakes +of all those other aspects, is profoundly and thoroughly POLITICAL. + +In 1990, the obscure, long-simmering struggle over the ownership +and nature of cyberspace became loudly and irretrievably public. +People from some of the oddest corners of American society suddenly +found themselves public figures. Some of these people found this +situation much more than they had ever bargained for. They backpedalled, +and tried to retreat back to the mandarin obscurity of their cozy +subcultural niches. This was generally to prove a mistake. + +But the civil libertarians seized the day in 1990. They found themselves +organizing, propagandizing, podium-pounding, persuading, touring, +negotiating, posing for publicity photos, submitting to interviews, +squinting in the limelight as they tried a tentative, but growingly +sophisticated, buck-and-wing upon the public stage. + +It's not hard to see why the civil libertarians should have +this competitive advantage. + +The hackers of the digital underground are an hermetic elite. +They find it hard to make any remotely convincing case for +their actions in front of the general public. Actually, +hackers roundly despise the "ignorant" public, and have never +trusted the judgement of "the system." Hackers do propagandize, +but only among themselves, mostly in giddy, badly spelled manifestos +of class warfare, youth rebellion or naive techie utopianism. +Hackers must strut and boast in order to establish and preserve +their underground reputations. But if they speak out too loudly +and publicly, they will break the fragile surface-tension of the underground, +and they will be harrassed or arrested. Over the longer term, +most hackers stumble, get busted, get betrayed, or simply give up. +As a political force, the digital underground is hamstrung. + +The telcos, for their part, are an ivory tower under protracted seige. +They have plenty of money with which to push their calculated public image, +but they waste much energy and goodwill attacking one another with +slanderous and demeaning ad campaigns. The telcos have suffered +at the hands of politicians, and, like hackers, they don't trust +the public's judgement. And this distrust may be well-founded. +Should the general public of the high-tech 1990s come to understand +its own best interests in telecommunications, that might well pose +a grave threat to the specialized technical power and authority +that the telcos have relished for over a century. The telcos do +have strong advantages: loyal employees, specialized expertise, +influence in the halls of power, tactical allies in law enforcement, +and unbelievably vast amounts of money. But politically speaking, they lack +genuine grassroots support; they simply don't seem to have many friends. + +Cops know a lot of things other people don't know. +But cops willingly reveal only those aspects of their +knowledge that they feel will meet their institutional +purposes and further public order. Cops have respect, +they have responsibilities, they have power in the streets +and even power in the home, but cops don't do particularly +well in limelight. When pressed, they will step out in the +public gaze to threaten bad-guys, or to cajole prominent citizens, +or perhaps to sternly lecture the naive and misguided. +But then they go back within their time-honored fortress +of the station-house, the courtroom and the rule-book. + +The electronic civil libertarians, however, have proven to be +born political animals. They seemed to grasp very early on +the postmodern truism that communication is power. Publicity is power. +Soundbites are power. The ability to shove one's issue onto the public +agenda--and KEEP IT THERE--is power. Fame is power. Simple personal +fluency and eloquence can be power, if you can somehow catch the +public's eye and ear. + +The civil libertarians had no monopoly on "technical power"-- +though they all owned computers, most were not particularly +advanced computer experts. They had a good deal of money, +but nowhere near the earthshaking wealth and the galaxy +of resources possessed by telcos or federal agencies. +They had no ability to arrest people. They carried +out no phreak and hacker covert dirty-tricks. + +But they really knew how to network. + +Unlike the other groups in this book, the civil libertarians +have operated very much in the open, more or less right +in the public hurly-burly. They have lectured audiences galore +and talked to countless journalists, and have learned to +refine their spiels. They've kept the cameras clicking, +kept those faxes humming, swapped that email, +run those photocopiers on overtime, licked envelopes +and spent small fortunes on airfare and long-distance. +In an information society, this open, overt, obvious activity +has proven to be a profound advantage. + +In 1990, the civil libertarians of cyberspace assembled +out of nowhere in particular, at warp speed. This "group" +(actually, a networking gaggle of interested parties +which scarcely deserves even that loose term) has almost nothing +in the way of formal organization. Those formal civil libertarian +organizations which did take an interest in cyberspace issues, +mainly the Computer Professionals for Social Responsibility +and the American Civil Liberties Union, were carried along +by events in 1990, and acted mostly as adjuncts, +underwriters or launching-pads. + +The civil libertarians nevertheless enjoyed the greatest success +of any of the groups in the Crackdown of 1990. At this writing, +their future looks rosy and the political initiative is firmly in their hands. +This should be kept in mind as we study the highly unlikely lives +and lifestyles of the people who actually made this happen. + +# + +In June 1989, Apple Computer, Inc., of Cupertino, +California, had a problem. Someone had illicitly copied +a small piece of Apple's proprietary software, software +which controlled an internal chip driving the Macintosh +screen display. This Color QuickDraw source code was +a closely guarded piece of Apple's intellectual property. +Only trusted Apple insiders were supposed to possess it. + +But the "NuPrometheus League" wanted things otherwise. +This person (or persons) made several illicit copies +of this source code, perhaps as many as two dozen. +He (or she, or they) then put those illicit floppy disks +into envelopes and mailed them to people all over America: +people in the computer industry who were associated with, +but not directly employed by, Apple Computer. + +The NuPrometheus caper was a complex, highly ideological, +and very hacker-like crime. Prometheus, it will be recalled, +stole the fire of the Gods and gave this potent gift to the +general ranks of downtrodden mankind. A similar god-in-the-manger +attitude was implied for the corporate elite of Apple Computer, +while the "Nu" Prometheus had himself cast in the role of rebel demigod. +The illicitly copied data was given away for free. + +The new Prometheus, whoever he was, escaped the +fate of the ancient Greek Prometheus, who was chained +to a rock for centuries by the vengeful gods while an eagle +tore and ate his liver. On the other hand, NuPrometheus +chickened out somewhat by comparison with his role model. +The small chunk of Color QuickDraw code he had filched +and replicated was more or less useless to Apple's +industrial rivals (or, in fact, to anyone else). +Instead of giving fire to mankind, it was more as if +NuPrometheus had photocopied the schematics for part of a Bic lighter. +The act was not a genuine work of industrial espionage. +It was best interpreted as a symbolic, deliberate slap +in the face for the Apple corporate heirarchy. + +Apple's internal struggles were well-known in the industry. Apple's founders, +Jobs and Wozniak, had both taken their leave long since. Their raucous core +of senior employees had been a barnstorming crew of 1960s Californians, +many of them markedly less than happy with the new button-down multimillion +dollar regime at Apple. Many of the programmers and developers who had +invented the Macintosh model in the early 1980s had also taken their leave of +the company. It was they, not the current masters of Apple's corporate fate, +who had invented the stolen Color QuickDraw code. The NuPrometheus stunt +was well-calculated to wound company morale. + +Apple called the FBI. The Bureau takes an interest in high-profile +intellectual-property theft cases, industrial espionage and theft +of trade secrets. These were likely the right people to call, +and rumor has it that the entities responsible were in fact discovered +by the FBI, and then quietly squelched by Apple management. NuPrometheus +was never publicly charged with a crime, or prosecuted, or jailed. +But there were no further illicit releases of Macintosh internal software. +Eventually the painful issue of NuPrometheus was allowed to fade. + +In the meantime, however, a large number of puzzled bystanders +found themselves entertaining surprise guests from the FBI. + +One of these people was John Perry Barlow. Barlow is a most unusual man, +difficult to describe in conventional terms. He is perhaps best known as +a songwriter for the Grateful Dead, for he composed lyrics for +"Hell in a Bucket," "Picasso Moon," "Mexicali Blues," "I Need a Miracle," +and many more; he has been writing for the band since 1970. + +Before we tackle the vexing question as to why a rock lyricist +should be interviewed by the FBI in a computer-crime case, +it might be well to say a word or two about the Grateful Dead. +The Grateful Dead are perhaps the most successful and long-lasting +of the numerous cultural emanations from the Haight-Ashbury district +of San Francisco, in the glory days of Movement politics and +lysergic transcendance. The Grateful Dead are a nexus, a veritable +whirlwind, of applique decals, psychedelic vans, tie-dyed T-shirts, +earth-color denim, frenzied dancing and open and unashamed drug use. +The symbols, and the realities, of Californian freak power surround +the Grateful Dead like knotted macrame. + +The Grateful Dead and their thousands of Deadhead devotees +are radical Bohemians. This much is widely understood. +Exactly what this implies in the 1990s is rather more problematic. + +The Grateful Dead are among the world's most popular +and wealthy entertainers: number 20, according to Forbes magazine, +right between M.C. Hammer and Sean Connery. In 1990, this jeans-clad +group of purported raffish outcasts earned seventeen million dollars. +They have been earning sums much along this line for quite some time now. + +And while the Dead are not investment bankers or three-piece-suit +tax specialists--they are, in point of fact, hippie musicians-- +this money has not been squandered in senseless Bohemian excess. +The Dead have been quietly active for many years, funding various +worthy activities in their extensive and widespread cultural community. + +The Grateful Dead are not conventional players in the American +power establishment. They nevertheless are something of a force +to be reckoned with. They have a lot of money and a lot of friends +in many places, both likely and unlikely. + +The Dead may be known for back-to-the-earth environmentalist rhetoric, +but this hardly makes them anti-technological Luddites. On the contrary, +like most rock musicians, the Grateful Dead have spent their entire adult +lives in the company of complex electronic equipment. They have funds to burn +on any sophisticated tool and toy that might happen to catch their fancy. +And their fancy is quite extensive. + +The Deadhead community boasts any number of recording engineers, +lighting experts, rock video mavens, electronic technicians +of all descriptions. And the drift goes both ways. Steve Wozniak, +Apple's co-founder, used to throw rock festivals. Silicon Valley rocks out. + +These are the 1990s, not the 1960s. Today, for a surprising number of people +all over America, the supposed dividing line between Bohemian and technician +simply no longer exists. People of this sort may have a set of windchimes +and a dog with a knotted kerchief 'round its neck, but they're also quite +likely to own a multimegabyte Macintosh running MIDI synthesizer software +and trippy fractal simulations. These days, even Timothy Leary himself, +prophet of LSD, does virtual-reality computer-graphics demos in +his lecture tours. + +John Perry Barlow is not a member of the Grateful Dead. He is, however, +a ranking Deadhead. + +Barlow describes himself as a "techno-crank." A vague term like +"social activist" might not be far from the mark, either. +But Barlow might be better described as a "poet"--if one keeps in mind +Percy Shelley's archaic definition of poets as "unacknowledged legislators +of the world." + +Barlow once made a stab at acknowledged legislator status. In 1987, +he narrowly missed the Republican nomination for a seat in the +Wyoming State Senate. Barlow is a Wyoming native, the third-generation +scion of a well-to-do cattle-ranching family. He is in his early forties, +married and the father of three daughters. + +Barlow is not much troubled by other people's narrow notions of consistency. +In the late 1980s, this Republican rock lyricist cattle rancher sold his ranch +and became a computer telecommunications devotee. + +The free-spirited Barlow made this transition with ease. He genuinely +enjoyed computers. With a beep of his modem, he leapt from small-town +Pinedale, Wyoming, into electronic contact with a large and lively crowd +of bright, inventive, technological sophisticates from all over the world. +Barlow found the social milieu of computing attractive: its fast-lane pace, +its blue-sky rhetoric, its open-endedness. Barlow began dabbling in +computer journalism, with marked success, as he was a quick study, +and both shrewd and eloquent. He frequently travelled to San Francisco +to network with Deadhead friends. There Barlow made extensive contacts +throughout the Californian computer community, including friendships +among the wilder spirits at Apple. + +In May 1990, Barlow received a visit from a local Wyoming agent of the FBI. +The NuPrometheus case had reached Wyoming. + +Barlow was troubled to find himself under investigation in an +area of his interests once quite free of federal attention. +He had to struggle to explain the very nature of computer-crime +to a headscratching local FBI man who specialized in cattle-rustling. +Barlow, chatting helpfully and demonstrating the wonders of his modem +to the puzzled fed, was alarmed to find all "hackers" generally under +FBI suspicion as an evil influence in the electronic community. +The FBI, in pursuit of a hacker called "NuPrometheus," were tracing +attendees of a suspect group called the Hackers Conference. + +The Hackers Conference, which had been started in 1984, was a +yearly Californian meeting of digital pioneers and enthusiasts. +The hackers of the Hackers Conference had little if anything to do +with the hackers of the digital underground. On the contrary, +the hackers of this conference were mostly well-to-do Californian +high-tech CEOs, consultants, journalists and entrepreneurs. +(This group of hackers were the exact sort of "hackers" +most likely to react with militant fury at any criminal +degradation of the term "hacker.") + +Barlow, though he was not arrested or accused of a crime, +and though his computer had certainly not gone out the door, +was very troubled by this anomaly. He carried the word to the Well. + +Like the Hackers Conference, "the Well" was an emanation of the +Point Foundation. Point Foundation, the inspiration of a wealthy +Californian 60s radical named Stewart Brand, was to be a major +launch-pad of the civil libertarian effort. + +Point Foundation's cultural efforts, like those of their fellow Bay Area +Californians the Grateful Dead, were multifaceted and multitudinous. +Rigid ideological consistency had never been a strong suit of the +Whole Earth Catalog. This Point publication had enjoyed a strong +vogue during the late 60s and early 70s, when it offered hundreds +of practical (and not so practical) tips on communitarian living, +environmentalism, and getting back-to-the-land. The Whole Earth Catalog, +and its sequels, sold two and half million copies and won a +National Book Award. + +With the slow collapse of American radical dissent, the Whole Earth Catalog +had slipped to a more modest corner of the cultural radar; but in its +magazine incarnation, CoEvolution Quarterly, the Point Foundation +continued to offer a magpie potpourri of "access to tools and ideas." + +CoEvolution Quarterly, which started in 1974, was never a widely +popular magazine. Despite periodic outbreaks of millenarian fervor, +CoEvolution Quarterly failed to revolutionize Western civilization +and replace leaden centuries of history with bright new Californian paradigms. +Instead, this propaganda arm of Point Foundation cakewalked a fine line between +impressive brilliance and New Age flakiness. CoEvolution Quarterly carried +no advertising, cost a lot, and came out on cheap newsprint with modest +black-and-white graphics. It was poorly distributed, and spread mostly +by subscription and word of mouth. + +It could not seem to grow beyond 30,000 subscribers. +And yet--it never seemed to shrink much, either. +Year in, year out, decade in, decade out, some strange +demographic minority accreted to support the magazine. +The enthusiastic readership did not seem to have much +in the way of coherent politics or ideals. It was sometimes +hard to understand what held them together (if the often bitter +debate in the letter-columns could be described as "togetherness"). + +But if the magazine did not flourish, it was resilient; it got by. +Then, in 1984, the birth-year of the Macintosh computer, +CoEvolution Quarterly suddenly hit the rapids. Point Foundation +had discovered the computer revolution. Out came the Whole Earth +Software Catalog of 1984, arousing headscratching doubts among +the tie-dyed faithful, and rabid enthusiasm among the nascent +"cyberpunk" milieu, present company included. Point Foundation +started its yearly Hackers Conference, and began to take an +extensive interest in the strange new possibilities of +digital counterculture. CoEvolution Quarterlyfolded its teepee, +replaced by Whole Earth Software Review and eventually by Whole Earth +Review (the magazine's present incarnation, currently under +the editorship of virtual-reality maven Howard Rheingold). + +1985 saw the birth of the "WELL"--the "Whole Earth 'Lectronic Link." +The Well was Point Foundation's bulletin board system. + +As boards went, the Well was an anomaly from the beginning, +and remained one. It was local to San Francisco. +It was huge, with multiple phonelines and enormous files +of commentary. Its complex UNIX-based software might be +most charitably described as "user-opaque." It was run on +a mainframe out of the rambling offices of a non-profit +cultural foundation in Sausalito. And it was crammed with +fans of the Grateful Dead. + +Though the Well was peopled by chattering hipsters of the Bay Area +counterculture, it was by no means a "digital underground" board. +Teenagers were fairly scarce; most Well users (known as "Wellbeings") +were thirty- and forty-something Baby Boomers. They tended to work +in the information industry: hardware, software, telecommunications, +media, entertainment. Librarians, academics, and journalists were +especially common on the Well, attracted by Point Foundation's +open-handed distribution of "tools and ideas." + +There were no anarchy files on the Well, scarcely a +dropped hint about access codes or credit-card theft. +No one used handles. Vicious "flame-wars" were held to +a comparatively civilized rumble. Debates were sometimes sharp, +but no Wellbeing ever claimed that a rival had disconnected his phone, +trashed his house, or posted his credit card numbers. + +The Well grew slowly as the 1980s advanced. It charged a modest sum +for access and storage, and lost money for years--but not enough to hamper +the Point Foundation, which was nonprofit anyway. By 1990, the Well +had about five thousand users. These users wandered about a gigantic +cyberspace smorgasbord of "Conferences", each conference itself consisting +of a welter of "topics," each topic containing dozens, sometimes hundreds +of comments, in a tumbling, multiperson debate that could last for months +or years on end. + + +In 1991, the Well's list of conferences looked like this: + + +CONFERENCES ON THE WELL + +WELL "Screenzine" Digest (g zine) + +Best of the WELL - vintage material - (g best) + +Index listing of new topics in all conferences - (g newtops) + +Business - Education +---------------------- + +Apple Library Users Group(g alug) Agriculture (g agri) +Brainstorming (g brain) Classifieds (g cla) +Computer Journalism (g cj) Consultants (g consult) +Consumers (g cons) Design (g design) +Desktop Publishing (g desk) Disability (g disability) +Education (g ed) Energy (g energy91) +Entrepreneurs (g entre) Homeowners (g home) +Indexing (g indexing) Investments (g invest) +Kids91 (g kids) Legal (g legal) +One Person Business (g one) +Periodical/newsletter (g per) +Telecomm Law (g tcl) The Future (g fut) +Translators (g trans) Travel (g tra) +Work (g work) + +Electronic Frontier Foundation (g eff) +Computers, Freedom & Privacy (g cfp) +Computer Professionals for Social Responsibility (g cpsr) + +Social - Political - Humanities +--------------------------------- + +Aging (g gray) AIDS (g aids) +Amnesty International (g amnesty) Archives (g arc) +Berkeley (g berk) Buddhist (g wonderland) +Christian (g cross) Couples (g couples) +Current Events (g curr) Dreams (g dream) +Drugs (g dru) East Coast (g east) +Emotional Health@@@@ (g private) Erotica (g eros) +Environment (g env) Firearms (g firearms) +First Amendment (g first) Fringes of Reason (g fringes) +Gay (g gay) Gay (Private)# (g gaypriv) +Geography (g geo) German (g german) +Gulf War (g gulf) Hawaii (g aloha) +Health (g heal) History (g hist) +Holistic (g holi) Interview (g inter) +Italian (g ital) Jewish (g jew) +Liberty (g liberty) Mind (g mind) +Miscellaneous (g misc) Men on the WELL@@ (g mow) +Network Integration (g origin) Nonprofits (g non) +North Bay (g north) Northwest (g nw) +Pacific Rim (g pacrim) Parenting (g par) +Peace (g pea) Peninsula (g pen) +Poetry (g poetry) Philosophy (g phi) +Politics (g pol) Psychology (g psy) +Psychotherapy (g therapy) Recovery## (g recovery) +San Francisco (g sanfran) Scams (g scam) +Sexuality (g sex) Singles (g singles) +Southern (g south) Spanish (g spanish) +Spirituality (g spirit) Tibet (g tibet) +Transportation (g transport) True Confessions (g tru) +Unclear (g unclear) WELL Writer's Workshop@@@(g www) +Whole Earth (g we) Women on the WELL@(g wow) +Words (g words) Writers (g wri) + +@@@@Private Conference - mail wooly for entry +@@@Private conference - mail sonia for entry +@@Private conference - mail flash for entry +@ Private conference - mail reva for entry +# Private Conference - mail hudu for entry +## Private Conference - mail dhawk for entry + +Arts - Recreation - Entertainment +----------------------------------- +ArtCom Electronic Net (g acen) +Audio-Videophilia (g aud) +Bicycles (g bike) Bay Area Tonight@@(g bat) +Boating (g wet) Books (g books) +CD's (g cd) Comics (g comics) +Cooking (g cook) Flying (g flying) +Fun (g fun) Games (g games) +Gardening (g gard) Kids (g kids) +Nightowls@ (g owl) Jokes (g jokes) +MIDI (g midi) Movies (g movies) +Motorcycling (g ride) Motoring (g car) +Music (g mus) On Stage (g onstage) +Pets (g pets) Radio (g rad) +Restaurant (g rest) Science Fiction (g sf) +Sports (g spo) Star Trek (g trek) +Television (g tv) Theater (g theater) +Weird (g weird) Zines/Factsheet Five(g f5) +@Open from midnight to 6am +@@Updated daily + +Grateful Dead +------------- +Grateful Dead (g gd) Deadplan@ (g dp) +Deadlit (g deadlit) Feedback (g feedback) +GD Hour (g gdh) Tapes (g tapes) +Tickets (g tix) Tours (g tours) + +@Private conference - mail tnf for entry + +Computers +----------- +AI/Forth/Realtime (g realtime) Amiga (g amiga) +Apple (g app) Computer Books (g cbook) +Art & Graphics (g gra) Hacking (g hack) +HyperCard (g hype) IBM PC (g ibm) +LANs (g lan) Laptop (g lap) +Macintosh (g mac) Mactech (g mactech) +Microtimes (g microx) Muchomedia (g mucho) +NeXt (g next) OS/2 (g os2) +Printers (g print) Programmer's Net (g net) +Siggraph (g siggraph) Software Design (g sdc) +Software/Programming (g software) +Software Support (g ssc) +Unix (g unix) Windows (g windows) +Word Processing (g word) + +Technical - Communications +---------------------------- +Bioinfo (g bioinfo) Info (g boing) +Media (g media) NAPLPS (g naplps) +Netweaver (g netweaver) Networld (g networld) +Packet Radio (g packet) Photography (g pho) +Radio (g rad) Science (g science) +Technical Writers (g tec) Telecommunications(g tele) +Usenet (g usenet) Video (g vid) +Virtual Reality (g vr) + +The WELL Itself +--------------- +Deeper (g deeper) Entry (g ent) +General (g gentech) Help (g help) +Hosts (g hosts) Policy (g policy) +System News (g news) Test (g test) + +The list itself is dazzling, bringing to the untutored eye +a dizzying impression of a bizarre milieu of mountain-climbing +Hawaiian holistic photographers trading true-life confessions +with bisexual word-processing Tibetans. + +But this confusion is more apparent than real. Each of these conferences +was a little cyberspace world in itself, comprising dozens and perhaps +hundreds of sub-topics. Each conference was commonly frequented by +a fairly small, fairly like-minded community of perhaps a few dozen people. +It was humanly impossible to encompass the entire Well (especially since +access to the Well's mainframe computer was billed by the hour). +Most long-time users contented themselves with a few favorite +topical neighborhoods, with the occasional foray elsewhere +for a taste of exotica. But especially important news items, +and hot topical debates, could catch the attention of the entire +Well community. + +Like any community, the Well had its celebrities, and John Perry Barlow, +the silver-tongued and silver-modemed lyricist of the Grateful Dead, +ranked prominently among them. It was here on the Well that Barlow +posted his true-life tale of computer-crime encounter with the FBI. + +The story, as might be expected, created a great stir. The Well was +already primed for hacker controversy. In December 1989, Harper's magazine +had hosted a debate on the Well about the ethics of illicit computer intrusion. +While over forty various computer-mavens took part, Barlow proved a star +in the debate. So did "Acid Phreak" and "Phiber Optik," a pair of young +New York hacker-phreaks whose skills at telco switching-station intrusion +were matched only by their apparently limitless hunger for fame. +The advent of these two boldly swaggering outlaws in the precincts +of the Well created a sensation akin to that of Black Panthers +at a cocktail party for the radically chic. + +Phiber Optik in particular was to seize the day in 1990. +A devotee of the 2600 circle and stalwart of the New York +hackers' group "Masters of Deception," Phiber Optik was +a splendid exemplar of the computer intruder as committed dissident. +The eighteen-year-old Optik, a high-school dropout and part-time +computer repairman, was young, smart, and ruthlessly obsessive, +a sharp-dressing, sharp-talking digital dude who was utterly +and airily contemptuous of anyone's rules but his own. +By late 1991, Phiber Optik had appeared in Harper's, +Esquire, The New York Times, in countless public debates +and conventions, even on a television show hosted by Geraldo Rivera. + +Treated with gingerly respect by Barlow and other Well mavens, +Phiber Optik swiftly became a Well celebrity. Strangely, despite +his thorny attitude and utter single-mindedness, Phiber Optik seemed +to arouse strong protective instincts in most of the people who met him. +He was great copy for journalists, always fearlessly ready to swagger, +and, better yet, to actually DEMONSTRATE some off-the-wall digital stunt. +He was a born media darling. + +Even cops seemed to recognize that there was something peculiarly unworldly +and uncriminal about this particular troublemaker. He was so bold, +so flagrant, so young, and so obviously doomed, that even those +who strongly disapproved of his actions grew anxious for his welfare, +and began to flutter about him as if he were an endangered seal pup. + +In January 24, 1990 (nine days after the Martin Luther King Day Crash), +Phiber Optik, Acid Phreak, and a third NYC scofflaw named Scorpion were +raided by the Secret Service. Their computers went out the door, +along with the usual blizzard of papers, notebooks, compact disks, +answering machines, Sony Walkmans, etc. Both Acid Phreak and +Phiber Optik were accused of having caused the Crash. + +The mills of justice ground slowly. The case eventually fell into +the hands of the New York State Police. Phiber had lost his machinery +in the raid, but there were no charges filed against him for over a year. +His predicament was extensively publicized on the Well, where it caused +much resentment for police tactics. It's one thing to merely hear about +a hacker raided or busted; it's another to see the police attacking someone +you've come to know personally, and who has explained his motives at length. +Through the Harper's debate on the Well, it had become clear to the +Wellbeings that Phiber Optik was not in fact going to "hurt anything." +In their own salad days, many Wellbeings had tasted tear-gas in pitched +street-battles with police. They were inclined to indulgence for +acts of civil disobedience. + +Wellbeings were also startled to learn of the draconian thoroughness +of a typical hacker search-and-seizure. It took no great stretch of +imagination for them to envision themselves suffering much the same treatment. + +As early as January 1990, sentiment on the Well had already begun to sour, +and people had begun to grumble that "hackers" were getting a raw deal +from the ham-handed powers-that-be. The resultant issue of Harper's +magazine posed the question as to whether computer-intrusion was a "crime" +at all. As Barlow put it later: "I've begun to wonder if we wouldn't +also regard spelunkers as desperate criminals if AT&T owned all the caves." + +In February 1991, more than a year after the raid on his home, +Phiber Optik was finally arrested, and was charged with first-degree +Computer Tampering and Computer Trespass, New York state offenses. +He was also charged with a theft-of-service misdemeanor, involving a complex +free-call scam to a 900 number. Phiber Optik pled guilty to the misdemeanor +charge, and was sentenced to 35 hours of community service. + +This passing harassment from the unfathomable world of straight people +seemed to bother Optik himself little if at all. Deprived of his computer +by the January search-and-seizure, he simply bought himself a portable +computer so the cops could no longer monitor the phone where he lived +with his Mom, and he went right on with his depredations, sometimes on +live radio or in front of television cameras. + +The crackdown raid may have done little to dissuade Phiber Optik, +but its galling affect on the Wellbeings was profound. As 1990 rolled on, +the slings and arrows mounted: the Knight Lightning raid, +the Steve Jackson raid, the nation-spanning Operation Sundevil. +The rhetoric of law enforcement made it clear that there was, +in fact, a concerted crackdown on hackers in progress. + +The hackers of the Hackers Conference, the Wellbeings, and their ilk, +did not really mind the occasional public misapprehension of "hacking;" +if anything, this membrane of differentiation from straight society +made the "computer community" feel different, smarter, better. +They had never before been confronted, however, by a concerted +vilification campaign. + +Barlow's central role in the counter-struggle was one of the major +anomalies of 1990. Journalists investigating the controversy +often stumbled over the truth about Barlow, but they commonly +dusted themselves off and hurried on as if nothing had happened. +It was as if it were TOO MUCH TO BELIEVE that a 1960s freak +from the Grateful Dead had taken on a federal law enforcement operation +head-to-head and ACTUALLY SEEMED TO BE WINNING! + +Barlow had no easily detectable power-base for a political struggle +of this kind. He had no formal legal or technical credentials. +Barlow was, however, a computer networker of truly stellar brilliance. +He had a poet's gift of concise, colorful phrasing. He also had a +journalist's shrewdness, an off-the-wall, self-deprecating wit, +and a phenomenal wealth of simple personal charm. + +The kind of influence Barlow possessed is fairly common currency +in literary, artistic, or musical circles. A gifted critic can +wield great artistic influence simply through defining +the temper of the times, by coining the catch-phrases +and the terms of debate that become the common currency of the period. +(And as it happened, Barlow WAS a part-time art critic, +with a special fondness for the Western art of Frederic Remington.) + +Barlow was the first commentator to adopt William Gibson's +striking science-fictional term "cyberspace" as a synonym +for the present-day nexus of computer and telecommunications networks. +Barlow was insistent that cyberspace should be regarded as +a qualitatively new world, a "frontier." According to Barlow, +the world of electronic communications, now made visible through +the computer screen, could no longer be usefully regarded +as just a tangle of high-tech wiring. Instead, it had become +a PLACE, cyberspace, which demanded a new set of metaphors, +a new set of rules and behaviors. The term, as Barlow employed it, +struck a useful chord, and this concept of cyberspace was picked up +by Time, Scientific American, computer police, hackers, and even +Constitutional scholars. "Cyberspace" now seems likely to become +a permanent fixture of the language. + +Barlow was very striking in person: a tall, craggy-faced, bearded, +deep-voiced Wyomingan in a dashing Western ensemble of jeans, jacket, +cowboy boots, a knotted throat-kerchief and an ever-present Grateful Dead +cloisonne lapel pin. + +Armed with a modem, however, Barlow was truly in his element. +Formal hierarchies were not Barlow's strong suit; he rarely missed +a chance to belittle the "large organizations and their drones," +with their uptight, institutional mindset. Barlow was very much +of the free-spirit persuasion, deeply unimpressed by brass-hats +and jacks-in-office. But when it came to the digital grapevine, +Barlow was a cyberspace ad-hocrat par excellence. + +There was not a mighty army of Barlows. There was only one Barlow, +and he was a fairly anomolous individual. However, the situation only +seemed to REQUIRE a single Barlow. In fact, after 1990, many people +must have concluded that a single Barlow was far more than +they'd ever bargained for. + +Barlow's querulous mini-essay about his encounter with the FBI +struck a strong chord on the Well. A number of other free spirits +on the fringes of Apple Computing had come under suspicion, +and they liked it not one whit better than he did. + +One of these was Mitchell Kapor, the co-inventor of the spreadsheet +program "Lotus 1-2-3" and the founder of Lotus Development Corporation. +Kapor had written-off the passing indignity of being fingerprinted +down at his own local Boston FBI headquarters, but Barlow's post +made the full national scope of the FBI's dragnet clear to Kapor. +The issue now had Kapor's full attention. As the Secret Service +swung into anti-hacker operation nationwide in 1990, Kapor watched +every move with deep skepticism and growing alarm. + +As it happened, Kapor had already met Barlow, who had interviewed Kapor +for a California computer journal. Like most people who met Barlow, +Kapor had been very taken with him. Now Kapor took it upon himself +to drop in on Barlow for a heart-to-heart talk about the situation. + +Kapor was a regular on the Well. Kapor had been a devotee of the +Whole Earth Catalogsince the beginning, and treasured a complete run +of the magazine. And Kapor not only had a modem, but a private jet. +In pursuit of the scattered high-tech investments of Kapor Enterprises Inc., +his personal, multi-million dollar holding company, Kapor commonly crossed +state lines with about as much thought as one might give to faxing a letter. + +The Kapor-Barlow council of June 1990, in Pinedale, Wyoming, was the start +of the Electronic Frontier Foundation. Barlow swiftly wrote a manifesto, +"Crime and Puzzlement," which announced his, and Kapor's, intention +to form a political organization to "raise and disburse funds for education, +lobbying, and litigation in the areas relating to digital speech and the +extension of the Constitution into Cyberspace." + +Furthermore, proclaimed the manifesto, the foundation would +"fund, conduct, and support legal efforts to demonstrate +that the Secret Service has exercised prior restraint on publications, +limited free speech, conducted improper seizure of equipment and data, +used undue force, and generally conducted itself in a fashion which +is arbitrary, oppressive, and unconstitutional." + +"Crime and Puzzlement" was distributed far and wide through computer +networking channels, and also printed in the Whole Earth Review. +The sudden declaration of a coherent, politicized counter-strike +from the ranks of hackerdom electrified the community. Steve Wozniak +(perhaps a bit stung by the NuPrometheus scandal) swiftly offered +to match any funds Kapor offered the Foundation. + +John Gilmore, one of the pioneers of Sun Microsystems, immediately offered +his own extensive financial and personal support. Gilmore, an ardent +libertarian, was to prove an eloquent advocate of electronic privacy issues, +especially freedom from governmental and corporate computer-assisted +surveillance of private citizens. + +A second meeting in San Francisco rounded up further allies: +Stewart Brand of the Point Foundation, virtual-reality pioneers +Jaron Lanier and Chuck Blanchard, network entrepreneur and venture +capitalist Nat Goldhaber. At this dinner meeting, the activists settled on +a formal title: the Electronic Frontier Foundation, Incorporated. +Kapor became its president. A new EFF Conference was opened on +the Point Foundation's Well, and the Well was declared +"the home of the Electronic Frontier Foundation." + +Press coverage was immediate and intense. Like their +nineteenth-century spiritual ancestors, Alexander Graham Bell +and Thomas Watson, the high-tech computer entrepreneurs +of the 1970s and 1980s--people such as Wozniak, Jobs, Kapor, +Gates, and H. Ross Perot, who had raised themselves by their bootstraps +to dominate a glittering new industry--had always made very good copy. + +But while the Wellbeings rejoiced, the press in general seemed +nonplussed by the self-declared "civilizers of cyberspace." +EFF's insistence that the war against "hackers" involved grave +Constitutional civil liberties issues seemed somewhat farfetched, +especially since none of EFF's organizers were lawyers +or established politicians. The business press in particular +found it easier to seize on the apparent core of the story-- +that high-tech entrepreneur Mitchell Kapor had established +a "defense fund for hackers." Was EFF a genuinely important +political development--or merely a clique of wealthy eccentrics, +dabbling in matters better left to the proper authorities? +The jury was still out. + +But the stage was now set for open confrontation. +And the first and the most critical battle was the +hacker show-trial of "Knight Lightning." + +# + +It has been my practice throughout this book to refer to hackers +only by their "handles." There is little to gain by giving +the real names of these people, many of whom are juveniles, +many of whom have never been convicted of any crime, and many +of whom had unsuspecting parents who have already suffered enough. + +But the trial of Knight Lightning on July 24-27, 1990, +made this particular "hacker" a nationally known public figure. +It can do no particular harm to himself or his family if I repeat +the long-established fact that his name is Craig Neidorf (pronounced NYE-dorf). + +Neidorf's jury trial took place in the United States District Court, +Northern District of Illinois, Eastern Division, with the +Honorable Nicholas J. Bua presiding. The United States of America +was the plaintiff, the defendant Mr. Neidorf. The defendant's attorney +was Sheldon T. Zenner of the Chicago firm of Katten, Muchin and Zavis. + +The prosecution was led by the stalwarts of the Chicago Computer Fraud +and Abuse Task Force: William J. Cook, Colleen D. Coughlin, and +David A. Glockner, all Assistant United States Attorneys. +The Secret Service Case Agent was Timothy M. Foley. + +It will be recalled that Neidorf was the co-editor of an underground hacker +"magazine" called Phrack. Phrack was an entirely electronic publication, +distributed through bulletin boards and over electronic networks. +It was amateur publication given away for free. Neidorf had never made +any money for his work in Phrack. Neither had his unindicted co-editor +"Taran King" or any of the numerous Phrack contributors. + +The Chicago Computer Fraud and Abuse Task Force, however, +had decided to prosecute Neidorf as a fraudster. +To formally admit that Phrack was a "magazine" +and Neidorf a "publisher" was to open a prosecutorial +Pandora's Box of First Amendment issues. To do this +was to play into the hands of Zenner and his EFF advisers, +which now included a phalanx of prominent New York civil rights +lawyers as well as the formidable legal staff of Katten, Muchin and Zavis. +Instead, the prosecution relied heavily on the issue of access device fraud: +Section 1029 of Title 18, the section from which the Secret Service drew +its most direct jurisdiction over computer crime. + +Neidorf's alleged crimes centered around the E911 Document. +He was accused of having entered into a fraudulent scheme with the Prophet, +who, it will be recalled, was the Atlanta LoD member who had illicitly +copied the E911 Document from the BellSouth AIMSX system. + +The Prophet himself was also a co-defendant in the Neidorf case, +part-and-parcel of the alleged "fraud scheme" to "steal" BellSouth's +E911 Document (and to pass the Document across state lines, +which helped establish the Neidorf trial as a federal case). +The Prophet, in the spirit of full co-operation, had agreed +to testify against Neidorf. + +In fact, all three of the Atlanta crew stood ready to testify against Neidorf. +Their own federal prosecutors in Atlanta had charged the Atlanta Three with: +(a) conspiracy, (b) computer fraud, (c) wire fraud, (d) access device fraud, +and (e) interstate transportation of stolen property (Title 18, Sections 371, +1030, 1343, 1029, and 2314). + +Faced with this blizzard of trouble, Prophet and Leftist had ducked +any public trial and had pled guilty to reduced charges--one conspiracy +count apiece. Urvile had pled guilty to that odd bit of Section 1029 +which makes it illegal to possess "fifteen or more" illegal access devices +(in his case, computer passwords). And their sentences were scheduled +for September 14, 1990--well after the Neidorf trial. As witnesses, +they could presumably be relied upon to behave. + +Neidorf, however, was pleading innocent. Most everyone else caught up +in the crackdown had "cooperated fully" and pled guilty in hope +of reduced sentences. (Steve Jackson was a notable exception, +of course, and had strongly protested his innocence from the +very beginning. But Steve Jackson could not get a day in court-- +Steve Jackson had never been charged with any crime in the first place.) + +Neidorf had been urged to plead guilty. But Neidorf was a political science +major and was disinclined to go to jail for "fraud" when he had not made +any money, had not broken into any computer, and had been publishing +a magazine that he considered protected under the First Amendment. + +Neidorf's trial was the ONLY legal action of the entire Crackdown +that actually involved bringing the issues at hand out for a public test +in front of a jury of American citizens. + +Neidorf, too, had cooperated with investigators. He had voluntarily +handed over much of the evidence that had led to his own indictment. +He had already admitted in writing that he knew that the E911 Document +had been stolen before he had "published" it in Phrack--or, from the +prosecution's point of view, illegally transported stolen property by wire +in something purporting to be a "publication." + +But even if the "publication" of the E911 Document was not held to be a crime, +that wouldn't let Neidorf off the hook. Neidorf had still received +the E911 Document when Prophet had transferred it to him from Rich Andrews' +Jolnet node. On that occasion, it certainly hadn't been "published"-- +it was hacker booty, pure and simple, transported across state lines. + +The Chicago Task Force led a Chicago grand jury to indict Neidorf +on a set of charges that could have put him in jail for thirty years. +When some of these charges were successfully challenged before Neidorf +actually went to trial, the Chicago Task Force rearranged his +indictment so that he faced a possible jail term of over sixty years! +As a first offender, it was very unlikely that Neidorf would in fact +receive a sentence so drastic; but the Chicago Task Force clearly +intended to see Neidorf put in prison, and his conspiratorial "magazine" +put permanently out of commission. This was a federal case, and Neidorf +was charged with the fraudulent theft of property worth almost +eighty thousand dollars. + +William Cook was a strong believer in high-profile prosecutions +with symbolic overtones. He often published articles on his work +in the security trade press, arguing that "a clear message had +to be sent to the public at large and the computer community +in particular that unauthorized attacks on computers and the theft +of computerized information would not be tolerated by the courts." + +The issues were complex, the prosecution's tactics somewhat unorthodox, +but the Chicago Task Force had proved sure-footed to date. "Shadowhawk" +had been bagged on the wing in 1989 by the Task Force, and sentenced +to nine months in prison, and a $10,000 fine. The Shadowhawk case involved +charges under Section 1030, the "federal interest computer" section. + +Shadowhawk had not in fact been a devotee of "federal-interest" computers +per se. On the contrary, Shadowhawk, who owned an AT&T home computer, +seemed to cherish a special aggression toward AT&T. He had bragged on +the underground boards "Phreak Klass 2600" and "Dr. Ripco" of his skills +at raiding AT&T, and of his intention to crash AT&T's national phone system. +Shadowhawk's brags were noticed by Henry Kluepfel of Bellcore Security, +scourge of the outlaw boards, whose relations with the Chicago Task Force +were long and intimate. + +The Task Force successfully established that Section 1030 applied to +the teenage Shadowhawk, despite the objections of his defense attorney. +Shadowhawk had entered a computer "owned" by U.S. Missile Command +and merely "managed" by AT&T. He had also entered an AT&T computer +located at Robbins Air Force Base in Georgia. Attacking AT&T was +of "federal interest" whether Shadowhawk had intended it or not. + +The Task Force also convinced the court that a piece of AT&T +software that Shadowhawk had illicitly copied from Bell Labs, +the "Artificial Intelligence C5 Expert System," was worth a cool +one million dollars. Shadowhawk's attorney had argued that +Shadowhawk had not sold the program and had made no profit from +the illicit copying. And in point of fact, the C5 Expert System +was experimental software, and had no established market value +because it had never been on the market in the first place. +AT&T's own assessment of a "one million dollar" figure for its +own intangible property was accepted without challenge +by the court, however. And the court concurred with +the government prosecutors that Shadowhawk showed clear +"intent to defraud" whether he'd gotten any money or not. +Shadowhawk went to jail. + +The Task Force's other best-known triumph had been the conviction +and jailing of "Kyrie." Kyrie, a true denizen of the digital +criminal underground, was a 36-year-old Canadian woman, +convicted and jailed for telecommunications fraud in Canada. +After her release from prison, she had fled the wrath of Canada Bell +and the Royal Canadian Mounted Police, and eventually settled, +very unwisely, in Chicago. + +"Kyrie," who also called herself "Long Distance Information," +specialized in voice-mail abuse. She assembled large numbers +of hot long-distance codes, then read them aloud into a series +of corporate voice-mail systems. Kyrie and her friends were +electronic squatters in corporate voice-mail systems, +using them much as if they were pirate bulletin boards, +then moving on when their vocal chatter clogged the system +and the owners necessarily wised up. Kyrie's camp followers +were a loose tribe of some hundred and fifty phone-phreaks, +who followed her trail of piracy from machine to machine, +ardently begging for her services and expertise. + +Kyrie's disciples passed her stolen credit-card numbers, +in exchange for her stolen "long distance information." +Some of Kyrie's clients paid her off in cash, by scamming +credit-card cash advances from Western Union. + +Kyrie travelled incessantly, mostly through airline tickets +and hotel rooms that she scammed through stolen credit cards. +Tiring of this, she found refuge with a fellow female phone +phreak in Chicago. Kyrie's hostess, like a surprising number +of phone phreaks, was blind. She was also physically disabled. +Kyrie allegedly made the best of her new situation by applying for, +and receiving, state welfare funds under a false identity as +a qualified caretaker for the handicapped. + +Sadly, Kyrie's two children by a former marriage had also vanished +underground with her; these pre-teen digital refugees had no legal +American identity, and had never spent a day in school. + +Kyrie was addicted to technical mastery and enthralled by her own +cleverness and the ardent worship of her teenage followers. +This foolishly led her to phone up Gail Thackeray in Arizona, +to boast, brag, strut, and offer to play informant. +Thackeray, however, had already learned far more +than enough about Kyrie, whom she roundly despised +as an adult criminal corrupting minors, a "female Fagin." +Thackeray passed her tapes of Kyrie's boasts to the Secret Service. + +Kyrie was raided and arrested in Chicago in May 1989. +She confessed at great length and pled guilty. + +In August 1990, Cook and his Task Force colleague Colleen Coughlin +sent Kyrie to jail for 27 months, for computer and telecommunications fraud. +This was a markedly severe sentence by the usual wrist-slapping standards +of "hacker" busts. Seven of Kyrie's foremost teenage disciples were also +indicted and convicted. The Kyrie "high-tech street gang," as Cook +described it, had been crushed. Cook and his colleagues had been +the first ever to put someone in prison for voice-mail abuse. +Their pioneering efforts had won them attention and kudos. + +In his article on Kyrie, Cook drove the message home to the readers +of Security Management magazine, a trade journal for corporate +security professionals. The case, Cook said, and Kyrie's stiff sentence, +"reflect a new reality for hackers and computer crime victims in the +'90s. . . . Individuals and corporations who report computer +and telecommunications crimes can now expect that their cooperation +with federal law enforcement will result in meaningful punishment. +Companies and the public at large must report computer-enhanced +crimes if they want prosecutors and the course to protect their rights +to the tangible and intangible property developed and stored on computers." + +Cook had made it his business to construct this "new reality for hackers." +He'd also made it his business to police corporate property rights +to the intangible. + +Had the Electronic Frontier Foundation been a "hacker defense fund" +as that term was generally understood, they presumably would have stood up +for Kyrie. Her 1990 sentence did indeed send a "message" that federal heat +was coming down on "hackers." But Kyrie found no defenders at EFF, +or anywhere else, for that matter. EFF was not a bail-out fund +for electronic crooks. + +The Neidorf case paralleled the Shadowhawk case in certain ways. +The victim once again was allowed to set the value of the "stolen" property. +Once again Kluepfel was both investigator and technical advisor. +Once again no money had changed hands, but the "intent to defraud" was central. + +The prosecution's case showed signs of weakness early on. The Task Force +had originally hoped to prove Neidorf the center of a nationwide +Legion of Doom criminal conspiracy. The Phrack editors threw physical +get-togethers every summer, which attracted hackers from across the country; +generally two dozen or so of the magazine's favorite contributors and readers. +(Such conventions were common in the hacker community; 2600 Magazine, +for instance, held public meetings of hackers in New York, every month.) +LoD heavy-dudes were always a strong presence at these Phrack-sponsored +"Summercons." + +In July 1988, an Arizona hacker named "Dictator" attended Summercon +in Neidorf's home town of St. Louis. Dictator was one of Gail Thackeray's +underground informants; Dictator's underground board in Phoenix was +a sting operation for the Secret Service. Dictator brought an undercover +crew of Secret Service agents to Summercon. The agents bored spyholes +through the wall of Dictator's hotel room in St Louis, and videotaped +the frolicking hackers through a one-way mirror. As it happened, +however, nothing illegal had occurred on videotape, other than the +guzzling of beer by a couple of minors. Summercons were social events, +not sinister cabals. The tapes showed fifteen hours of raucous laughter, +pizza-gobbling, in-jokes and back-slapping. + +Neidorf's lawyer, Sheldon Zenner, saw the Secret Service tapes +before the trial. Zenner was shocked by the complete harmlessness +of this meeting, which Cook had earlier characterized as a sinister +interstate conspiracy to commit fraud. Zenner wanted to show the +Summercon tapes to the jury. It took protracted maneuverings +by the Task Force to keep the tapes from the jury as "irrelevant." + +The E911 Document was also proving a weak reed. It had originally +been valued at $79,449. Unlike Shadowhawk's arcane Artificial Intelligence +booty, the E911 Document was not software--it was written in English. +Computer-knowledgeable people found this value--for a twelve-page +bureaucratic document--frankly incredible. In his "Crime and Puzzlement" +manifesto for EFF, Barlow commented: "We will probably never know how +this figure was reached or by whom, though I like to imagine an appraisal +team consisting of Franz Kafka, Joseph Heller, and Thomas Pynchon." + +As it happened, Barlow was unduly pessimistic. The EFF did, in fact, +eventually discover exactly how this figure was reached, and by whom-- +but only in 1991, long after the Neidorf trial was over. + +Kim Megahee, a Southern Bell security manager, +had arrived at the document's value by simply adding up +the "costs associated with the production" of the E911 Document. +Those "costs" were as follows: + +1. A technical writer had been hired to research and write the E911 Document. + 200 hours of work, at $35 an hour, cost : $7,000. A Project Manager had + overseen the technical writer. 200 hours, at $31 an hour, made: $6,200. + +2. A week of typing had cost $721 dollars. A week of formatting had + cost $721. A week of graphics formatting had cost $742. + +3. Two days of editing cost $367. + +4. A box of order labels cost five dollars. + +5. Preparing a purchase order for the Document, including typing + and the obtaining of an authorizing signature from within the + BellSouth bureaucracy, cost $129. + +6. Printing cost $313. Mailing the Document to fifty people + took fifty hours by a clerk, and cost $858. + +7. Placing the Document in an index took two clerks an hour each, + totalling $43. + +Bureaucratic overhead alone, therefore, was alleged to have cost +a whopping $17,099. According to Mr. Megahee, the typing +of a twelve-page document had taken a full week. Writing it +had taken five weeks, including an overseer who apparently +did nothing else but watch the author for five weeks. +Editing twelve pages had taken two days. Printing and mailing +an electronic document (which was already available on the +Southern Bell Data Network to any telco employee who needed it), +had cost over a thousand dollars. + +But this was just the beginning. There were also the HARDWARE EXPENSES. +Eight hundred fifty dollars for a VT220 computer monitor. +THIRTY-ONE THOUSAND DOLLARS for a sophisticated VAXstation II computer. +Six thousand dollars for a computer printer. TWENTY-TWO THOUSAND DOLLARS +for a copy of "Interleaf" software. Two thousand five hundred dollars +for VMS software. All this to create the twelve-page Document. + +Plus ten percent of the cost of the software and the hardware, for maintenance. +(Actually, the ten percent maintenance costs, though mentioned, had been left +off the final $79,449 total, apparently through a merciful oversight). + +Mr. Megahee's letter had been mailed directly to William Cook himself, +at the office of the Chicago federal attorneys. The United States Government +accepted these telco figures without question. + +As incredulity mounted, the value of the E911 Document was officially +revised downward. This time, Robert Kibler of BellSouth Security +estimated the value of the twelve pages as a mere $24,639.05--based, +purportedly, on "R&D costs." But this specific estimate, +right down to the nickel, did not move the skeptics at all; +in fact it provoked open scorn and a torrent of sarcasm. + +The financial issues concerning theft of proprietary information +have always been peculiar. It could be argued that BellSouth +had not "lost" its E911 Document at all in the first place, +and therefore had not suffered any monetary damage from this "theft." +And Sheldon Zenner did in fact argue this at Neidorf's trial-- +that Prophet's raid had not been "theft," but was better understood +as illicit copying. + +The money, however, was not central to anyone's true purposes in this trial. +It was not Cook's strategy to convince the jury that the E911 Document +was a major act of theft and should be punished for that reason alone. +His strategy was to argue that the E911 Document was DANGEROUS. +It was his intention to establish that the E911 Document was "a road-map" +to the Enhanced 911 System. Neidorf had deliberately and recklessly +distributed a dangerous weapon. Neidorf and the Prophet did not care +(or perhaps even gloated at the sinister idea) that the E911 Document +could be used by hackers to disrupt 911 service, "a life line for every +person certainly in the Southern Bell region of the United States, +and indeed, in many communities throughout the United States," +in Cook's own words. Neidorf had put people's lives in danger. + +In pre-trial maneuverings, Cook had established that the E911 Document +was too hot to appear in the public proceedings of the Neidorf trial. +The JURY ITSELF would not be allowed to ever see this Document, +lest it slip into the official court records, and thus into the hands +of the general public, and, thus, somehow, to malicious hackers +who might lethally abuse it. + +Hiding the E911 Document from the jury may have been a +clever legal maneuver, but it had a severe flaw. There were, +in point of fact, hundreds, perhaps thousands, of people, +already in possession of the E911 Document, just as Phrack +had published it. Its true nature was already obvious +to a wide section of the interested public (all of whom, +by the way, were, at least theoretically, party to +a gigantic wire-fraud conspiracy). Most everyone +in the electronic community who had a modem and any +interest in the Neidorf case already had a copy of the Document. +It had already been available in Phrack for over a year. + +People, even quite normal people without any particular +prurient interest in forbidden knowledge, did not shut their eyes +in terror at the thought of beholding a "dangerous" document +from a telephone company. On the contrary, they tended to trust +their own judgement and simply read the Document for themselves. +And they were not impressed. + +One such person was John Nagle. Nagle was a forty-one-year-old +professional programmer with a masters' degree in computer science +from Stanford. He had worked for Ford Aerospace, where he had invented +a computer-networking technique known as the "Nagle Algorithm," +and for the prominent Californian computer-graphics firm "Autodesk," +where he was a major stockholder. + +Nagle was also a prominent figure on the Well, much respected +for his technical knowledgeability. + +Nagle had followed the civil-liberties debate closely, +for he was an ardent telecommunicator. He was no particular friend +of computer intruders, but he believed electronic publishing +had a great deal to offer society at large, and attempts +to restrain its growth, or to censor free electronic expression, +strongly roused his ire. + +The Neidorf case, and the E911 Document, were both being discussed +in detail on the Internet, in an electronic publication called Telecom Digest. +Nagle, a longtime Internet maven, was a regular reader of Telecom Digest. +Nagle had never seen a copy of Phrack, but the implications of the case +disturbed him. + +While in a Stanford bookstore hunting books on robotics, +Nagle happened across a book called The Intelligent Network. +Thumbing through it at random, Nagle came across an entire chapter +meticulously detailing the workings of E911 police emergency systems. +This extensive text was being sold openly, and yet in Illinois +a young man was in danger of going to prison for publishing +a thin six-page document about 911 service. + +Nagle made an ironic comment to this effect in Telecom Digest. +From there, Nagle was put in touch with Mitch Kapor, +and then with Neidorf's lawyers. + +Sheldon Zenner was delighted to find a computer telecommunications expert +willing to speak up for Neidorf, one who was not a wacky teenage "hacker." +Nagle was fluent, mature, and respectable; he'd once had a federal +security clearance. + +Nagle was asked to fly to Illinois to join the defense team. + +Having joined the defense as an expert witness, Nagle read the entire +E911 Document for himself. He made his own judgement about its potential +for menace. + +The time has now come for you yourself, the reader, to have a look +at the E911 Document. This six-page piece of work was the pretext +for a federal prosecution that could have sent an electronic publisher +to prison for thirty, or even sixty, years. It was the pretext +for the search and seizure of Steve Jackson Games, a legitimate publisher +of printed books. It was also the formal pretext for the search +and seizure of the Mentor's bulletin board, "Phoenix Project," +and for the raid on the home of Erik Bloodaxe. It also had much +to do with the seizure of Richard Andrews' Jolnet node +and the shutdown of Charles Boykin's AT&T node. +The E911 Document was the single most important piece +of evidence in the Hacker Crackdown. There can be no real +and legitimate substitute for the Document itself. + + +==Phrack Inc.== + +Volume Two, Issue 24, File 5 of 13 + +Control Office Administration +Of Enhanced 911 Services For +Special Services and Account Centers + +by the Eavesdropper + +March, 1988 + + +Description of Service +~~~~~~~~~~~~~~~~~~~~~ +The control office for Emergency 911 service is assigned in +accordance with the existing standard guidelines to one of +the following centers: + +o Special Services Center (SSC) +o Major Accounts Center (MAC) +o Serving Test Center (STC) +o Toll Control Center (TCC) + +The SSC/MAC designation is used in this document interchangeably +for any of these four centers. The Special Services Centers (SSCs) +or Major Account Centers (MACs) have been designated as the trouble +reporting contact for all E911 customer (PSAP) reported troubles. +Subscribers who have trouble on an E911 call will continue +to contact local repair service (CRSAB) who will refer the +trouble to the SSC/MAC, when appropriate. + +Due to the critical nature of E911 service, the control +and timely repair of troubles is demanded. As the primary +E911 customer contact, the SSC/MAC is in the unique position +to monitor the status of the trouble and insure its resolution. + +System Overview +~~~~~~~~~~~~~~ +The number 911 is intended as a nationwide universal +telephone number which provides the public with direct +access to a Public Safety Answering Point (PSAP). A PSAP +is also referred to as an Emergency Service Bureau (ESB). +A PSAP is an agency or facility which is authorized by a +municipality to receive and respond to police, fire and/or +ambulance services. One or more attendants are located +at the PSAP facilities to receive and handle calls of an +emergency nature in accordance with the local municipal +requirements. + +An important advantage of E911 emergency service is +improved (reduced) response times for emergency +services. Also close coordination among agencies +providing various emergency services is a valuable +capability provided by E911 service. + +1A ESS is used as the tandem office for the E911 network to +route all 911 calls to the correct (primary) PSAP designated +to serve the calling station. The E911 feature was +developed primarily to provide routing to the correct PSAP +for all 911 calls. Selective routing allows a 911 call +originated from a particular station located in a particular +district, zone, or town, to be routed to the primary PSAP +designated to serve that customer station regardless of +wire center boundaries. Thus, selective routing eliminates +the problem of wire center boundaries not coinciding with +district or other political boundaries. + +The services available with the E911 feature include: + +Forced Disconnect Default Routing +Alternative Routing Night Service +Selective Routing Automatic Number +Identification (ANI) +Selective Transfer Automatic Location +Identification (ALI) + + +Preservice/Installation Guidelines +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +When a contract for an E911 system has been signed, it is +the responsibility of Network Marketing to establish an +implementation/cutover committee which should include +a representative from the SSC/MAC. Duties of the E911 +Implementation Team include coordination of all phases +of the E911 system deployment and the formation of an +on-going E911 maintenance subcommittee. + +Marketing is responsible for providing the following +customer specific information to the SSC/MAC prior to +the start of call through testing: + +o All PSAP's (name, address, local contact) +o All PSAP circuit ID's +o 1004 911 service request including PSAP details on each PSAP + (1004 Section K, L, M) +o Network configuration +o Any vendor information (name, telephone number, equipment) + +The SSC/MAC needs to know if the equipment and sets +at the PSAP are maintained by the BOCs, an independent +company, or an outside vendor, or any combination. +This information is then entered on the PSAP profile sheets +and reviewed quarterly for changes, additions and deletions. + +Marketing will secure the Major Account Number (MAN) +and provide this number to Corporate Communications +so that the initial issue of the service orders carry +the MAN and can be tracked by the SSC/MAC via CORDNET. +PSAP circuits are official services by definition. + +All service orders required for the installation of the E911 +system should include the MAN assigned to the city/county +which has purchased the system. + +In accordance with the basic SSC/MAC strategy for provisioning, +the SSC/MAC will be Overall Control Office (OCO) for all Node +to PSAP circuits (official services) and any other services +for this customer. Training must be scheduled for all SSC/MAC +involved personnel during the pre-service stage of the project. + +The E911 Implementation Team will form the on-going +maintenance subcommittee prior to the initial +implementation of the E911 system. This sub-committee +will establish post implementation quality assurance +procedures to ensure that the E911 system continues to +provide quality service to the customer. +Customer/Company training, trouble reporting interfaces +for the customer, telephone company and any involved +independent telephone companies needs to be addressed +and implemented prior to E911 cutover. These functions +can be best addressed by the formation of a sub- +committee of the E911 Implementation Team to set up +guidelines for and to secure service commitments of +interfacing organizations. A SSC/MAC supervisor should +chair this subcommittee and include the following +organizations: + +1) Switching Control Center + - E911 translations + - Trunking + - End office and Tandem office hardware/software +2) Recent Change Memory Administration Center + - Daily RC update activity for TN/ESN translations + - Processes validity errors and rejects +3) Line and Number Administration + - Verification of TN/ESN translations +4) Special Service Center/Major Account Center + - Single point of contact for all PSAP and Node to host troubles + - Logs, tracks & statusing of all trouble reports + - Trouble referral, follow up, and escalation + - Customer notification of status and restoration + - Analyzation of "chronic" troubles + - Testing, installation and maintenance of E911 circuits +5) Installation and Maintenance (SSIM/I&M) + - Repair and maintenance of PSAP equipment and Telco owned sets +6) Minicomputer Maintenance Operations Center + - E911 circuit maintenance (where applicable) +7) Area Maintenance Engineer + - Technical assistance on voice (CO-PSAP) network related E911 troubles + + +Maintenance Guidelines +~~~~~~~~~~~~~~~~~~~~~ +The CCNC will test the Node circuit from the 202T at the +Host site to the 202T at the Node site. Since Host to Node +(CCNC to MMOC) circuits are official company services, +the CCNC will refer all Node circuit troubles to the +SSC/MAC. The SSC/MAC is responsible for the testing +and follow up to restoration of these circuit troubles. + +Although Node to PSAP circuit are official services, the +MMOC will refer PSAP circuit troubles to the appropriate +SSC/MAC. The SSC/MAC is responsible for testing and +follow up to restoration of PSAP circuit troubles. + +The SSC/MAC will also receive reports from +CRSAB/IMC(s) on subscriber 911 troubles when they are +not line troubles. The SSC/MAC is responsible for testing +and restoration of these troubles. + +Maintenance responsibilities are as follows: + +SCC@ Voice Network (ANI to PSAP) +@SCC responsible for tandem switch + +SSIM/I&M PSAP Equipment (Modems, CIU's, sets) +Vendor PSAP Equipment (when CPE) +SSC/MAC PSAP to Node circuits, and tandem to + PSAP voice circuits (EMNT) +MMOC Node site (Modems, cables, etc) + +Note: All above work groups are required to resolve troubles +by interfacing with appropriate work groups for resolution. + +The Switching Control Center (SCC) is responsible for +E911/1AESS translations in tandem central offices. +These translations route E911 calls, selective transfer, +default routing, speed calling, etc., for each PSAP. +The SCC is also responsible for troubleshooting on +the voice network (call originating to end office tandem equipment). + +For example, ANI failures in the originating offices would +be a responsibility of the SCC. + +Recent Change Memory Administration Center (RCMAC) performs +the daily tandem translation updates (recent change) +for routing of individual telephone numbers. + +Recent changes are generated from service order activity +(new service, address changes, etc.) and compiled into +a daily file by the E911 Center (ALI/DMS E911 Computer). + +SSIM/I&M is responsible for the installation and repair of +PSAP equipment. PSAP equipment includes ANI Controller, +ALI Controller, data sets, cables, sets, and other peripheral +equipment that is not vendor owned. SSIM/I&M is responsible +for establishing maintenance test kits, complete with spare parts +for PSAP maintenance. This includes test gear, data sets, +and ANI/ALI Controller parts. + +Special Services Center (SSC) or Major Account Center +(MAC) serves as the trouble reporting contact for all +(PSAP) troubles reported by customer. The SSC/MAC +refers troubles to proper organizations for handling and +tracks status of troubles, escalating when necessary. +The SSC/MAC will close out troubles with customer. +The SSC/MAC will analyze all troubles and tracks "chronic" +PSAP troubles. + +Corporate Communications Network Center (CCNC) will +test and refer troubles on all node to host circuits. +All E911 circuits are classified as official company property. + +The Minicomputer Maintenance Operations Center +(MMOC) maintains the E911 (ALI/DMS) computer +hardware at the Host site. This MMOC is also responsible +for monitoring the system and reporting certain PSAP +and system problems to the local MMOC's, SCC's or +SSC/MAC's. The MMOC personnel also operate software +programs that maintain the TN data base under the +direction of the E911 Center. The maintenance of the +NODE computer (the interface between the PSAP and the +ALI/DMS computer) is a function of the MMOC at the +NODE site. The MMOC's at the NODE sites may also be +involved in the testing of NODE to Host circuits. +The MMOC will also assist on Host to PSAP and data network +related troubles not resolved through standard trouble +clearing procedures. + +Installation And Maintenance Center (IMC) is responsible +for referral of E911 subscriber troubles that are not subscriber +line problems. + +E911 Center - Performs the role of System Administration +and is responsible for overall operation of the E911 +computer software. The E911 Center does A-Z trouble +analysis and provides statistical information on the +performance of the system. + +This analysis includes processing PSAP inquiries (trouble +reports) and referral of network troubles. The E911 Center +also performs daily processing of tandem recent change +and provides information to the RCMAC for tandem input. +The E911 Center is responsible for daily processing +of the ALI/DMS computer data base and provides error files, +etc. to the Customer Services department for investigation and correction. +The E911 Center participates in all system implementations and on-going +maintenance effort and assists in the development of procedures, +training and education of information to all groups. + +Any group receiving a 911 trouble from the SSC/MAC should +close out the trouble with the SSC/MAC or provide a status +if the trouble has been referred to another group. +This will allow the SSC/MAC to provide a status back +to the customer or escalate as appropriate. + +Any group receiving a trouble from the Host site (MMOC +or CCNC) should close the trouble back to that group. + +The MMOC should notify the appropriate SSC/MAC +when the Host, Node, or all Node circuits are down so that +the SSC/MAC can reply to customer reports that may be +called in by the PSAPs. This will eliminate duplicate +reporting of troubles. On complete outages the MMOC +will follow escalation procedures for a Node after two (2) +hours and for a PSAP after four (4) hours. Additionally the +MMOC will notify the appropriate SSC/MAC when the +Host, Node, or all Node circuits are down. + +The PSAP will call the SSC/MAC to report E911 troubles. +The person reporting the E911 trouble may not have a +circuit I.D. and will therefore report the PSAP name and +address. Many PSAP troubles are not circuit specific. In +those instances where the caller cannot provide a circuit +I.D., the SSC/MAC will be required to determine the +circuit I.D. using the PSAP profile. Under no circumstances +will the SSC/MAC Center refuse to take the trouble. +The E911 trouble should be handled as quickly as possible, +with the SSC/MAC providing as much assistance as +possible while taking the trouble report from the caller. + +The SSC/MAC will screen/test the trouble to determine the +appropriate handoff organization based on the following criteria: + +PSAP equipment problem: SSIM/I&M +Circuit problem: SSC/MAC +Voice network problem: SCC (report trunk group number) +Problem affecting multiple PSAPs (No ALI report from +all PSAPs): Contact the MMOC to check for NODE or +Host computer problems before further testing. + +The SSC/MAC will track the status of reported troubles +and escalate as appropriate. The SSC/MAC will close out +customer/company reports with the initiating contact. +Groups with specific maintenance responsibilities, +defined above, will investigate "chronic" troubles upon +request from the SSC/MAC and the ongoing maintenance subcommittee. + +All "out of service" E911 troubles are priority one type reports. +One link down to a PSAP is considered a priority one trouble +and should be handled as if the PSAP was isolated. + +The PSAP will report troubles with the ANI controller, ALI +controller or set equipment to the SSC/MAC. + +NO ANI: Where the PSAP reports NO ANI (digital +display screen is blank) ask if this condition exists on all +screens and on all calls. It is important to differentiate +between blank screens and screens displaying 911-00XX, +or all zeroes. + +When the PSAP reports all screens on all calls, ask if there +is any voice contact with callers. If there is no voice +contact the trouble should be referred to the SCC +immediately since 911 calls are not getting through which +may require alternate routing of calls to another PSAP. + +When the PSAP reports this condition on all screens +but not all calls and has voice contact with callers, +the report should be referred to SSIM/I&M for dispatch. +The SSC/MAC should verify with the SCC that ANI +is pulsing before dispatching SSIM. + +When the PSAP reports this condition on one screen for +all calls (others work fine) the trouble should be referred +to SSIM/I&M for dispatch, because the trouble is isolated to +one piece of equipment at the customer premise. + +An ANI failure (i.e. all zeroes) indicates that the ANI has +not been received by the PSAP from the tandem office or +was lost by the PSAP ANI controller. The PSAP may +receive "02" alarms which can be caused by the ANI +controller logging more than three all zero failures on the +same trunk. The PSAP has been instructed to report this +condition to the SSC/MAC since it could indicate an +equipment trouble at the PSAP which might be affecting +all subscribers calling into the PSAP. When all zeroes are +being received on all calls or "02" alarms continue, a tester +should analyze the condition to determine the appropriate +action to be taken. The tester must perform cooperative +testing with the SCC when there appears to be a problem +on the Tandem-PSAP trunks before requesting dispatch. + +When an occasional all zero condition is reported, +the SSC/MAC should dispatch SSIM/I&M to routine +equipment on a "chronic" troublesweep. + +The PSAPs are instructed to report incidental ANI failures +to the BOC on a PSAP inquiry trouble ticket (paper) that +is sent to the Customer Services E911 group and forwarded +to E911 center when required. This usually involves only a +particular telephone number and is not a condition that +would require a report to the SSC/MAC. Multiple ANI +failures which our from the same end office (XX denotes +end office), indicate a hard trouble condition may exist +in the end office or end office tandem trunks. The PSAP will +report this type of condition to the SSC/MAC and the +SSC/MAC should refer the report to the SCC responsible +for the tandem office. NOTE: XX is the ESCO (Emergency +Service Number) associated with the incoming 911 trunks +into the tandem. It is important that the C/MAC tell the +SCC what is displayed at the PSAP (i.e. 911-0011) which +indicates to the SCC which end office is in trouble. + +Note: It is essential that the PSAP fill out inquiry form +on every ANI failure. + +The PSAP will report a trouble any time an address is not +received on an address display (screen blank) E911 call. +(If a record is not in the 911 data base or an ANI failure +is encountered, the screen will provide a display noticing +such condition). The SSC/MAC should verify with the PSAP +whether the NO ALI condition is on one screen or all screens. + +When the condition is on one screen (other screens +receive ALI information) the SSC/MAC will request +SSIM/I&M to dispatch. + +If no screens are receiving ALI information, there is usually +a circuit trouble between the PSAP and the Host computer. +The SSC/MAC should test the trouble and refer for restoral. + +Note: If the SSC/MAC receives calls from multiple +PSAP's, all of which are receiving NO ALI, there is a +problem with the Node or Node to Host circuits or the +Host computer itself. Before referring the trouble the +SSC/MAC should call the MMOC to inquire if the Node +or Host is in trouble. + +Alarm conditions on the ANI controller digital display at +the PSAP are to be reported by the PSAP's. These alarms +can indicate various trouble conditions so the SSC/MAC +should ask the PSAP if any portion of the E911 system +is not functioning properly. + +The SSC/MAC should verify with the PSAP attendant that +the equipment's primary function is answering E911 calls. +If it is, the SSC/MAC should request a dispatch SSIM/I&M. +If the equipment is not primarily used for E911, +then the SSC/MAC should advise PSAP to contact their CPE vendor. + +Note: These troubles can be quite confusing when the +PSAP has vendor equipment mixed in with equipment +that the BOC maintains. The Marketing representative +should provide the SSC/MAC information concerning any +unusual or exception items where the PSAP should +contact their vendor. This information should be included +in the PSAP profile sheets. + +ANI or ALI controller down: When the host computer sees +the PSAP equipment down and it does not come back up, +the MMOC will report the trouble to the SSC/MAC; +the equipment is down at the PSAP, a dispatch will be required. + +PSAP link (circuit) down: The MMOC will provide the +SSC/MAC with the circuit ID that the Host computer +indicates in trouble. Although each PSAP has two circuits, +when either circuit is down the condition must be treated +as an emergency since failure of the second circuit will +cause the PSAP to be isolated. + +Any problems that the MMOC identifies from the Node +location to the Host computer will be handled directly +with the appropriate MMOC(s)/CCNC. + +Note: The customer will call only when a problem is +apparent to the PSAP. When only one circuit is down to +the PSAP, the customer may not be aware there is a +trouble, even though there is one link down, +notification should appear on the PSAP screen. +Troubles called into the SSC/MAC from the MMOC +or other company employee should not be closed out +by calling the PSAP since it may result in the +customer responding that they do not have a trouble. +These reports can only be closed out by receiving +information that the trouble was fixed and by checking +with the company employee that reported the trouble. +The MMOC personnel will be able to verify that the +trouble has cleared by reviewing a printout from the host. + +When the CRSAB receives a subscriber complaint +(i.e., cannot dial 911) the RSA should obtain as much +information as possible while the customer is on the line. + +For example, what happened when the subscriber dialed 911? +The report is automatically directed to the IMC for subscriber line testing. +When no line trouble is found, the IMC will refer the trouble condition +to the SSC/MAC. The SSC/MAC will contact Customer Services E911 Group +and verify that the subscriber should be able to call 911 and obtain the ESN. +The SSC/MAC will verify the ESN via 2SCCS. When both verifications match, +the SSC/MAC will refer the report to the SCC responsible for the 911 tandem +office for investigation and resolution. The MAC is responsible for tracking +the trouble and informing the IMC when it is resolved. + + +For more information, please refer to E911 Glossary of Terms. +End of Phrack File +_____________________________________ + + +The reader is forgiven if he or she was entirely unable to read +this document. John Perry Barlow had a great deal of fun at its expense, +in "Crime and Puzzlement:" "Bureaucrat-ese of surpassing opacity. . . . +To read the whole thing straight through without entering coma requires +either a machine or a human who has too much practice thinking like one. +Anyone who can understand it fully and fluidly had altered his consciousness +beyond the ability to ever again read Blake, Whitman, or Tolstoy. . . . +the document contains little of interest to anyone who is not a student +of advanced organizational sclerosis." + +With the Document itself to hand, however, exactly as it was published +(in its six-page edited form) in Phrack, the reader may be able to verify +a few statements of fact about its nature. First, there is no software, +no computer code, in the Document. It is not computer-programming language +like FORTRAN or C++, it is English; all the sentences have nouns and verbs +and punctuation. It does not explain how to break into the E911 system. +It does not suggest ways to destroy or damage the E911 system. + +There are no access codes in the Document. There are no computer passwords. +It does not explain how to steal long distance service. It does not explain +how to break in to telco switching stations. There is nothing in it about +using a personal computer or a modem for any purpose at all, good or bad. + +Close study will reveal that this document is not about machinery. +The E911 Document is about ADMINISTRATION. It describes how one creates +and administers certain units of telco bureaucracy: +Special Service Centers and Major Account Centers (SSC/MAC). +It describes how these centers should distribute responsibility +for the E911 service, to other units of telco bureaucracy, +in a chain of command, a formal hierarchy. It describes +who answers customer complaints, who screens calls, +who reports equipment failures, who answers those reports, +who handles maintenance, who chairs subcommittees, +who gives orders, who follows orders, WHO tells WHOM what to do. +The Document is not a "roadmap" to computers. +The Document is a roadmap to PEOPLE. + +As an aid to breaking into computer systems, the Document is USELESS. +As an aid to harassing and deceiving telco people, however, the Document +might prove handy (especially with its Glossary, which I have not included). +An intense and protracted study of this Document and its Glossary, +combined with many other such documents, might teach one to speak like +a telco employee. And telco people live by SPEECH--they live by phone +communication. If you can mimic their language over the phone, +you can "social-engineer" them. If you can con telco people, you can +wreak havoc among them. You can force them to no longer trust one another; +you can break the telephonic ties that bind their community; you can make +them paranoid. And people will fight harder to defend their community +than they will fight to defend their individual selves. + +This was the genuine, gut-level threat posed by Phrack magazine. +The real struggle was over the control of telco language, +the control of telco knowledge. It was a struggle to defend the social +"membrane of differentiation" that forms the walls of the telco +community's ivory tower --the special jargon that allows telco +professionals to recognize one another, and to exclude charlatans, +thieves, and upstarts. And the prosecution brought out this fact. +They repeatedly made reference to the threat posed to telco professionals +by hackers using "social engineering." + +However, Craig Neidorf was not on trial for learning to speak like +a professional telecommunications expert. Craig Neidorf was on trial +for access device fraud and transportation of stolen property. +He was on trial for stealing a document that was purportedly +highly sensitive and purportedly worth tens of thousands of dollars. + +# + +John Nagle read the E911 Document. He drew his own conclusions. +And he presented Zenner and his defense team with an overflowing box +of similar material, drawn mostly from Stanford University's +engineering libraries. During the trial, the defense team--Zenner, +half-a-dozen other attorneys, Nagle, Neidorf, and computer-security +expert Dorothy Denning, all pored over the E911 Document line-by-line. + +On the afternoon of July 25, 1990, Zenner began to cross-examine +a woman named Billie Williams, a service manager for Southern Bell +in Atlanta. Ms. Williams had been responsible for the E911 Document. +(She was not its author--its original "author" was a Southern Bell +staff manager named Richard Helms. However, Mr. Helms should not bear +the entire blame; many telco staff people and maintenance personnel +had amended the Document. It had not been so much "written" by a +single author, as built by committee out of concrete-blocks of jargon.) + +Ms. Williams had been called as a witness for the prosecution, +and had gamely tried to explain the basic technical structure +of the E911 system, aided by charts. + +Now it was Zenner's turn. He first established that the +"proprietary stamp" that BellSouth had used on the E911 Document +was stamped on EVERY SINGLE DOCUMENT that BellSouth wrote-- +THOUSANDS of documents. "We do not publish anything other +than for our own company," Ms. Williams explained. +"Any company document of this nature is considered proprietary." +Nobody was in charge of singling out special high-security publications +for special high-security protection. They were ALL special, +no matter how trivial, no matter what their subject matter-- +the stamp was put on as soon as any document was written, +and the stamp was never removed. + +Zenner now asked whether the charts she had been using to explain +the mechanics of E911 system were "proprietary," too. +Were they PUBLIC INFORMATION, these charts, all about PSAPs, +ALIs, nodes, local end switches? Could he take the charts out +in the street and show them to anybody, "without violating +some proprietary notion that BellSouth has?" + +Ms Williams showed some confusion, but finally areed that the charts were, +in fact, public. + +"But isn't this what you said was basically what appeared in Phrack?" + +Ms. Williams denied this. + +Zenner now pointed out that the E911 Document as published in Phrack +was only half the size of the original E911 Document (as Prophet +had purloined it). Half of it had been deleted--edited by Neidorf. + +Ms. Williams countered that "Most of the information that is +in the text file is redundant." + +Zenner continued to probe. Exactly what bits of knowledge in the Document +were, in fact, unknown to the public? Locations of E911 computers? +Phone numbers for telco personnel? Ongoing maintenance subcommittees? +Hadn't Neidorf removed much of this? + +Then he pounced. "Are you familiar with Bellcore Technical Reference +Document TR-TSY-000350?" It was, Zenner explained, officially titled +"E911 Public Safety Answering Point Interface Between 1-1AESS Switch +and Customer Premises Equipment." It contained highly detailed +and specific technical information about the E911 System. +It was published by Bellcore and publicly available for about $20. + +He showed the witness a Bellcore catalog which listed thousands +of documents from Bellcore and from all the Baby Bells, BellSouth included. +The catalog, Zenner pointed out, was free. Anyone with a credit card +could call the Bellcore toll-free 800 number and simply order any +of these documents, which would be shipped to any customer without question. +Including, for instance, "BellSouth E911 Service Interfaces to +Customer Premises Equipment at a Public Safety Answering Point." + +Zenner gave the witness a copy of "BellSouth E911 Service Interfaces," +which cost, as he pointed out, $13, straight from the catalog. +"Look at it carefully," he urged Ms. Williams, "and tell me +if it doesn't contain about twice as much detailed information +about the E911 system of BellSouth than appeared anywhere in Phrack." + +"You want me to. . . ." Ms. Williams trailed off. "I don't understand." + +"Take a careful look," Zenner persisted. "Take a look at that document, +and tell me when you're done looking at it if, indeed, it doesn't contain +much more detailed information about the E911 system than appeared in Phrack." + +"Phrack wasn't taken from this," Ms. Williams said. + +"Excuse me?" said Zenner. + +"Phrack wasn't taken from this." + +"I can't hear you," Zenner said. + +"Phrack was not taken from this document. I don't understand +your question to me." + +"I guess you don't," Zenner said. + +At this point, the prosecution's case had been gutshot. +Ms. Williams was distressed. Her confusion was quite genuine. +Phrack had not been taken from any publicly available Bellcore document. +Phrack's E911 Document had been stolen from her own company's computers, +from her own company's text files, that her own colleagues had written, +and revised, with much labor. + +But the "value" of the Document had been blown to smithereens. +It wasn't worth eighty grand. According to Bellcore it was worth +thirteen bucks. And the looming menace that it supposedly posed +had been reduced in instants to a scarecrow. Bellcore itself +was selling material far more detailed and "dangerous," +to anybody with a credit card and a phone. + +Actually, Bellcore was not giving this information to just anybody. +They gave it to ANYBODY WHO ASKED, but not many did ask. +Not many people knew that Bellcore had a free catalog and an 800 number. +John Nagle knew, but certainly the average teenage phreak didn't know. +"Tuc," a friend of Neidorf's and sometime Phrack contributor, knew, +and Tuc had been very helpful to the defense, behind the scenes. +But the Legion of Doom didn't know--otherwise, they would never +have wasted so much time raiding dumpsters. Cook didn't know. +Foley didn't know. Kluepfel didn't know. The right hand +of Bellcore knew not what the left hand was doing. The right +hand was battering hackers without mercy, while the left hand +was distributing Bellcore's intellectual property to anybody +who was interested in telephone technical trivia--apparently, +a pathetic few. + +The digital underground was so amateurish and poorly organized +that they had never discovered this heap of unguarded riches. +The ivory tower of the telcos was so wrapped-up in the fog +of its own technical obscurity that it had left all the +windows open and flung open the doors. No one had even noticed. + +Zenner sank another nail in the coffin. He produced a printed issue +of Telephone Engineer & Management, a prominent industry journal +that comes out twice a month and costs $27 a year. This particular issue +of TE&M, called "Update on 911," featured a galaxy of technical details +on 911 service and a glossary far more extensive than Phrack's. + +The trial rumbled on, somehow, through its own momentum. +Tim Foley testified about his interrogations of Neidorf. +Neidorf's written admission that he had known the E911 Document +was pilfered was officially read into the court record. + +An interesting side issue came up: "Terminus" had once passed Neidorf +a piece of UNIX AT&T software, a log-in sequence, that had been cunningly +altered so that it could trap passwords. The UNIX software itself was +illegally copied AT&T property, and the alterations "Terminus" had made to it, +had transformed it into a device for facilitating computer break-ins. Terminus +himself would eventually plead guilty to theft of this piece of software, +and the Chicago group would send Terminus to prison for it. But it was +of dubious relevance in the Neidorf case. Neidorf hadn't written the program. +He wasn't accused of ever having used it. And Neidorf wasn't being charged +with software theft or owning a password trapper. + +On the next day, Zenner took the offensive. The civil libertarians +now had their own arcane, untried legal weaponry to launch into action-- +the Electronic Communications Privacy Act of 1986, 18 US Code, +Section 2701 et seq. Section 2701 makes it a crime to intentionally +access without authorization a facility in which an electronic communication +service is provided--it is, at heart, an anti-bugging and anti-tapping law, +intended to carry the traditional protections of telephones into other +electronic channels of communication. While providing penalties for amateur +snoops, however, Section 2703 of the ECPA also lays some formal difficulties +on the bugging and tapping activities of police. + +The Secret Service, in the person of Tim Foley, had served Richard Andrews +with a federal grand jury subpoena, in their pursuit of Prophet, +the E911 Document, and the Terminus software ring. But according to +the Electronic Communications Privacy Act, a "provider of remote +computing service" was legally entitled to "prior notice" from +the government if a subpoena was used. Richard Andrews and his +basement UNIX node, Jolnet, had not received any "prior notice." +Tim Foley had purportedly violated the ECPA and committed +an electronic crime! Zenner now sought the judge's permission +to cross-examine Foley on the topic of Foley's own electronic misdeeds. + +Cook argued that Richard Andrews' Jolnet was a privately owned +bulletin board, and not within the purview of ECPA. Judge Bua +granted the motion of the government to prevent cross-examination +on that point, and Zenner's offensive fizzled. This, however, +was the first direct assault on the legality of the actions +of the Computer Fraud and Abuse Task Force itself-- +the first suggestion that they themselves had broken the law, +and might, perhaps, be called to account. + +Zenner, in any case, did not really need the ECPA. +Instead, he grilled Foley on the glaring contradictions in +the supposed value of the E911 Document. He also brought up +the embarrassing fact that the supposedly red-hot E911 Document +had been sitting around for months, in Jolnet, with Kluepfel's knowledge, +while Kluepfel had done nothing about it. + +In the afternoon, the Prophet was brought in to testify +for the prosecution. (The Prophet, it will be recalled, +had also been indicted in the case as partner in a fraud +scheme with Neidorf.) In Atlanta, the Prophet had already +pled guilty to one charge of conspiracy, one charge of wire fraud +and one charge of interstate transportation of stolen property. +The wire fraud charge, and the stolen property charge, +were both directly based on the E911 Document. + +The twenty-year-old Prophet proved a sorry customer, +answering questions politely but in a barely audible mumble, +his voice trailing off at the ends of sentences. +He was constantly urged to speak up. + +Cook, examining Prophet, forced him to admit that +he had once had a "drug problem," abusing amphetamines, +marijuana, cocaine, and LSD. This may have established +to the jury that "hackers" are, or can be, seedy lowlife characters, +but it may have damaged Prophet's credibility somewhat. +Zenner later suggested that drugs might have damaged Prophet's memory. +The interesting fact also surfaced that Prophet had never +physically met Craig Neidorf. He didn't even know +Neidorf's last name--at least, not until the trial. + +Prophet confirmed the basic facts of his hacker career. +He was a member of the Legion of Doom. He had abused codes, +he had broken into switching stations and re-routed calls, +he had hung out on pirate bulletin boards. He had raided +the BellSouth AIMSX computer, copied the E911 Document, +stored it on Jolnet, mailed it to Neidorf. He and Neidorf +had edited it, and Neidorf had known where it came from. + +Zenner, however, had Prophet confirm that Neidorf was not a member +of the Legion of Doom, and had not urged Prophet to break into +BellSouth computers. Neidorf had never urged Prophet to defraud anyone, +or to steal anything. Prophet also admitted that he had never known Neidorf +to break in to any computer. Prophet said that no one in the Legion of Doom +considered Craig Neidorf a "hacker" at all. Neidorf was not a UNIX maven, +and simply lacked the necessary skill and ability to break into computers. +Neidorf just published a magazine. + +On Friday, July 27, 1990, the case against Neidorf collapsed. +Cook moved to dismiss the indictment, citing "information currently +available to us that was not available to us at the inception of the trial." +Judge Bua praised the prosecution for this action, which he described as +"very responsible," then dismissed a juror and declared a mistrial. + +Neidorf was a free man. His defense, however, had cost himself +and his family dearly. Months of his life had been consumed in anguish; +he had seen his closest friends shun him as a federal criminal. +He owed his lawyers over a hundred thousand dollars, despite +a generous payment to the defense by Mitch Kapor. + +Neidorf was not found innocent. The trial was simply dropped. +Nevertheless, on September 9, 1991, Judge Bua granted Neidorf's +motion for the "expungement and sealing" of his indictment record. +The United States Secret Service was ordered to delete and destroy +all fingerprints, photographs, and other records of arrest +or processing relating to Neidorf's indictment, including +their paper documents and their computer records. + +Neidorf went back to school, blazingly determined to become a lawyer. +Having seen the justice system at work, Neidorf lost much of his enthusiasm +for merely technical power. At this writing, Craig Neidorf is working +in Washington as a salaried researcher for the American Civil Liberties Union. + +# + +The outcome of the Neidorf trial changed the EFF +from voices-in-the-wilderness to the media darlings +of the new frontier. + +Legally speaking, the Neidorf case was not a sweeping triumph +for anyone concerned. No constitutional principles had been established. +The issues of "freedom of the press" for electronic publishers remained +in legal limbo. There were public misconceptions about the case. +Many people thought Neidorf had been found innocent and relieved +of all his legal debts by Kapor. The truth was that the government +had simply dropped the case, and Neidorf's family had gone deeply +into hock to support him. + +But the Neidorf case did provide a single, devastating, public sound-bite: +THE FEDS SAID IT WAS WORTH EIGHTY GRAND, AND IT WAS ONLY WORTH THIRTEEN BUCKS. + +This is the Neidorf case's single most memorable element. No serious report +of the case missed this particular element. Even cops could not read this +without a wince and a shake of the head. It left the public credibility +of the crackdown agents in tatters. + +The crackdown, in fact, continued, however. Those two charges +against Prophet, which had been based on the E911 Document, +were quietly forgotten at his sentencing--even though Prophet +had already pled guilty to them. Georgia federal prosecutors +strongly argued for jail time for the Atlanta Three, insisting on +"the need to send a message to the community," "the message that +hackers around the country need to hear." + +There was a great deal in their sentencing memorandum +about the awful things that various other hackers had done +(though the Atlanta Three themselves had not, in fact, +actually committed these crimes). There was also much +speculation about the awful things that the Atlanta Three +MIGHT have done and WERE CAPABLE of doing (even though +they had not, in fact, actually done them). +The prosecution's argument carried the day. +The Atlanta Three were sent to prison: +Urvile and Leftist both got 14 months each, +while Prophet (a second offender) got 21 months. + +The Atlanta Three were also assessed staggering fines as "restitution": +$233,000 each. BellSouth claimed that the defendants had "stolen" +"approximately $233,880 worth" of "proprietary computer access information"-- +specifically, $233,880 worth of computer passwords and connect addresses. +BellSouth's astonishing claim of the extreme value of its own computer +passwords and addresses was accepted at face value by the Georgia court. +Furthermore (as if to emphasize its theoretical nature) this enormous sum +was not divvied up among the Atlanta Three, but each of them had to pay +all of it. + +A striking aspect of the sentence was that the Atlanta Three were +specifically forbidden to use computers, except for work or under supervision. +Depriving hackers of home computers and modems makes some sense if one +considers hackers as "computer addicts," but EFF, filing an amicus brief +in the case, protested that this punishment was unconstitutional-- +it deprived the Atlanta Three of their rights of free association +and free expression through electronic media. + +Terminus, the "ultimate hacker," was finally sent to prison for a year +through the dogged efforts of the Chicago Task Force. His crime, +to which he pled guilty, was the transfer of the UNIX password trapper, +which was officially valued by AT&T at $77,000, a figure which aroused +intense skepticism among those familiar with UNIX "login.c" programs. + +The jailing of Terminus and the Atlanta Legionnaires of Doom, however, +did not cause the EFF any sense of embarrassment or defeat. +On the contrary, the civil libertarians were rapidly gathering strength. + +An early and potent supporter was Senator Patrick Leahy, +Democrat from Vermont, who had been a Senate sponsor +of the Electronic Communications Privacy Act. Even before +the Neidorf trial, Leahy had spoken out in defense of hacker-power +and freedom of the keyboard: "We cannot unduly inhibit the inquisitive +13-year-old who, if left to experiment today, may tomorrow develop +the telecommunications or computer technology to lead the United States +into the 21st century. He represents our future and our best hope +to remain a technologically competitive nation." + +It was a handsome statement, rendered perhaps rather more effective +by the fact that the crackdown raiders DID NOT HAVE any Senators +speaking out for THEM. On the contrary, their highly secretive +actions and tactics, all "sealed search warrants" here and +"confidential ongoing investigations" there, might have won +them a burst of glamorous publicity at first, but were crippling +them in the on-going propaganda war. Gail Thackeray was reduced +to unsupported bluster: "Some of these people who are loudest +on the bandwagon may just slink into the background," +she predicted in Newsweek--when all the facts came out, +and the cops were vindicated. + +But all the facts did not come out. Those facts that did, +were not very flattering. And the cops were not vindicated. +And Gail Thackeray lost her job. By the end of 1991, +William Cook had also left public employment. + +1990 had belonged to the crackdown, but by '91 its agents +were in severe disarray, and the libertarians were on a roll. +People were flocking to the cause. + +A particularly interesting ally had been Mike Godwin of Austin, Texas. +Godwin was an individual almost as difficult to describe as Barlow; +he had been editor of the student newspaper of the University of Texas, +and a computer salesman, and a programmer, and in 1990 was back +in law school, looking for a law degree. + +Godwin was also a bulletin board maven. He was very well-known +in the Austin board community under his handle "Johnny Mnemonic," +which he adopted from a cyberpunk science fiction story by William Gibson. +Godwin was an ardent cyberpunk science fiction fan. As a fellow Austinite +of similar age and similar interests, I myself had known Godwin socially +for many years. When William Gibson and myself had been writing our +collaborative SF novel, The Difference Engine, Godwin had been our +technical advisor in our effort to link our Apple word-processors +from Austin to Vancouver. Gibson and I were so pleased by his generous +expert help that we named a character in the novel "Michael Godwin" +in his honor. + +The handle "Mnemonic" suited Godwin very well. His erudition +and his mastery of trivia were impressive to the point of stupor; +his ardent curiosity seemed insatiable, and his desire to debate +and argue seemed the central drive of his life. Godwin had even +started his own Austin debating society, wryly known as the +"Dull Men's Club." In person, Godwin could be overwhelming; +a flypaper-brained polymath who could not seem to let any idea go. +On bulletin boards, however, Godwin's closely reasoned, +highly grammatical, erudite posts suited the medium well, +and he became a local board celebrity. + +Mike Godwin was the man most responsible for the public national exposure +of the Steve Jackson case. The Izenberg seizure in Austin had received +no press coverage at all. The March 1 raids on Mentor, Bloodaxe, and +Steve Jackson Games had received a brief front-page splash in the +front page of the Austin American-Statesman, but it was confused +and ill-informed: the warrants were sealed, and the Secret Service +wasn't talking. Steve Jackson seemed doomed to obscurity. +Jackson had not been arrested; he was not charged with any crime; +he was not on trial. He had lost some computers in an ongoing +investigation--so what? Jackson tried hard to attract attention +to the true extent of his plight, but he was drawing a blank; +no one in a position to help him seemed able to get a mental grip +on the issues. + +Godwin, however, was uniquely, almost magically, qualified +to carry Jackson's case to the outside world. Godwin was +a board enthusiast, a science fiction fan, a former journalist, +a computer salesman, a lawyer-to-be, and an Austinite. +Through a coincidence yet more amazing, in his last year +of law school Godwin had specialized in federal prosecutions +and criminal procedure. Acting entirely on his own, Godwin made +up a press packet which summarized the issues and provided useful +contacts for reporters. Godwin's behind-the-scenes effort +(which he carried out mostly to prove a point in a local board debate) +broke the story again in the Austin American-Statesman and then in Newsweek. + +Life was never the same for Mike Godwin after that. As he joined the growing +civil liberties debate on the Internet, it was obvious to all parties involved +that here was one guy who, in the midst of complete murk and confusion, +GENUINELY UNDERSTOOD EVERYTHING HE WAS TALKING ABOUT. The disparate elements +of Godwin's dilettantish existence suddenly fell together as neatly as +the facets of a Rubik's cube. + +When the time came to hire a full-time EFF staff attorney, +Godwin was the obvious choice. He took the Texas bar exam, +left Austin, moved to Cambridge, became a full-time, professional, +computer civil libertarian, and was soon touring the nation on behalf +of EFF, delivering well-received addresses on the issues to crowds +as disparate as academics, industrialists, science fiction fans, +and federal cops. + +Michael Godwin is currently the chief legal counsel of +the Electronic Frontier Foundation in Cambridge, Massachusetts. + +# + +Another early and influential participant in the controversy +was Dorothy Denning. Dr. Denning was unique among investigators +of the computer underground in that she did not enter the debate +with any set of politicized motives. She was a professional +cryptographer and computer security expert whose primary interest +in hackers was SCHOLARLY. She had a B.A. and M.A. in mathematics, +and a Ph.D. in computer science from Purdue. She had worked for SRI +International, the California think-tank that was also the home of +computer-security maven Donn Parker, and had authored an influential text +called Cryptography and Data Security. In 1990, Dr. Denning was working for +Digital Equipment Corporation in their Systems Reseach Center. Her husband, +Peter Denning, was also a computer security expert, working for NASA's +Research Institute for Advanced Computer Science. He had edited the +well-received Computers Under Attack: Intruders, Worms and Viruses. + +Dr. Denning took it upon herself to contact the digital underground, +more or less with an anthropological interest. There she discovered +that these computer-intruding hackers, who had been characterized +as unethical, irresponsible, and a serious danger to society, +did in fact have their own subculture and their own rules. +They were not particularly well-considered rules, but they were, +in fact, rules. Basically, they didn't take money and they +didn't break anything. + +Her dispassionate reports on her researches did a great deal +to influence serious-minded computer professionals--the sort +of people who merely rolled their eyes at the cyberspace +rhapsodies of a John Perry Barlow. + +For young hackers of the digital underground, meeting Dorothy Denning +was a genuinely mind-boggling experience. Here was this neatly coiffed, +conservatively dressed, dainty little personage, who reminded most +hackers of their moms or their aunts. And yet she was an IBM systems +programmer with profound expertise in computer architectures +and high-security information flow, who had personal friends +in the FBI and the National Security Agency. + +Dorothy Denning was a shining example of the American mathematical +intelligentsia, a genuinely brilliant person from the central ranks +of the computer-science elite. And here she was, gently questioning +twenty-year-old hairy-eyed phone-phreaks over the deeper ethical +implications of their behavior. + +Confronted by this genuinely nice lady, most hackers sat up very straight +and did their best to keep the anarchy-file stuff down to a faint whiff +of brimstone. Nevertheless, the hackers WERE in fact prepared to seriously +discuss serious issues with Dorothy Denning. They were willing to speak +the unspeakable and defend the indefensible, to blurt out their convictions +that information cannot be owned, that the databases of governments and large +corporations were a threat to the rights and privacy of individuals. + +Denning's articles made it clear to many that "hacking" +was not simple vandalism by some evil clique of psychotics. +"Hacking" was not an aberrant menace that could be charmed away +by ignoring it, or swept out of existence by jailing a few ringleaders. +Instead, "hacking" was symptomatic of a growing, primal struggle over +knowledge and power in the age of information. + +Denning pointed out that the attitude of hackers were at least partially +shared by forward-looking management theorists in the business community: +people like Peter Drucker and Tom Peters. Peter Drucker, in his book +The New Realities, had stated that "control of information by the government +is no longer possible. Indeed, information is now transnational. +Like money, it has no `fatherland.'" + +And management maven Tom Peters had chided large corporations for uptight, +proprietary attitudes in his bestseller, Thriving on Chaos: +"Information hoarding, especially by politically motivated, +power-seeking staffs, had been commonplace throughout American industry, +service and manufacturing alike. It will be an impossible +millstone aroung the neck of tomorrow's organizations." + +Dorothy Denning had shattered the social membrane of the +digital underground. She attended the Neidorf trial, +where she was prepared to testify for the defense as an expert witness. +She was a behind-the-scenes organizer of two of the most important +national meetings of the computer civil libertarians. Though not +a zealot of any description, she brought disparate elements of the +electronic community into a surprising and fruitful collusion. + +Dorothy Denning is currently the Chair of the Computer Science Department +at Georgetown University in Washington, DC. + +# + +There were many stellar figures in the civil libertarian community. +There's no question, however, that its single most influential figure +was Mitchell D. Kapor. Other people might have formal titles, +or governmental positions, have more experience with crime, +or with the law, or with the arcanities of computer security +or constitutional theory. But by 1991 Kapor had transcended +any such narrow role. Kapor had become "Mitch." + +Mitch had become the central civil-libertarian ad-hocrat. +Mitch had stood up first, he had spoken out loudly, directly, +vigorously and angrily, he had put his own reputation, +and his very considerable personal fortune, on the line. +By mid-'91 Kapor was the best-known advocate of his cause +and was known PERSONALLY by almost every single human being in America +with any direct influence on the question of civil liberties in cyberspace. +Mitch had built bridges, crossed voids, changed paradigms, forged metaphors, +made phone-calls and swapped business cards to such spectacular effect +that it had become impossible for anyone to take any action in the +"hacker question" without wondering what Mitch might think-- +and say--and tell his friends. + +The EFF had simply NETWORKED the situation into an entirely new status quo. +And in fact this had been EFF's deliberate strategy from the beginning. +Both Barlow and Kapor loathed bureaucracies and had deliberately +chosen to work almost entirely through the electronic spiderweb of +"valuable personal contacts." + +After a year of EFF, both Barlow and Kapor had every reason +to look back with satisfaction. EFF had established its own Internet node, +"eff.org," with a well-stocked electronic archive of documents on +electronic civil rights, privacy issues, and academic freedom. +EFF was also publishing EFFector, a quarterly printed journal, +as well as EFFector Online, an electronic newsletter with +over 1,200 subscribers. And EFF was thriving on the Well. + +EFF had a national headquarters in Cambridge and a full-time staff. +It had become a membership organization and was attracting +grass-roots support. It had also attracted the support +of some thirty civil-rights lawyers, ready and eager +to do pro bono work in defense of the Constitution in Cyberspace. + +EFF had lobbied successfully in Washington and in Massachusetts +to change state and federal legislation on computer networking. +Kapor in particular had become a veteran expert witness, +and had joined the Computer Science and Telecommunications Board +of the National Academy of Science and Engineering. + +EFF had sponsored meetings such as "Computers, Freedom and Privacy" +and the CPSR Roundtable. It had carried out a press offensive that, +in the words of EFFector, "has affected the climate of opinion about +computer networking and begun to reverse the slide into +`hacker hysteria' that was beginning to grip the nation." + +It had helped Craig Neidorf avoid prison. + +And, last but certainly not least, the Electronic Frontier Foundation +had filed a federal lawsuit in the name of Steve Jackson, +Steve Jackson Games Inc., and three users of the Illuminati +bulletin board system. The defendants were, and are, +the United States Secret Service, William Cook, Tim Foley, +Barbara Golden and Henry Kleupfel. + +The case, which is in pre-trial procedures in an Austin federal court +as of this writing, is a civil action for damages to redress +alleged violations of the First and Fourth Amendments to the +United States Constitution, as well as the Privacy Protection Act +of 1980 (42 USC 2000aa et seq.), and the Electronic Communications +Privacy Act (18 USC 2510 et seq and 2701 et seq). + +EFF had established that it had credibility. It had also established +that it had teeth. + +In the fall of 1991 I travelled to Massachusetts to speak personally +with Mitch Kapor. It was my final interview for this book. + +# + +The city of Boston has always been one of the major intellectual centers +of the American republic. It is a very old city by American standards, +a place of skyscrapers overshadowing seventeenth-century graveyards, +where the high-tech start-up companies of Route 128 co-exist with the +hand-wrought pre-industrial grace of "Old Ironsides," the USS CONSTITUTION. + +The Battle of Bunker Hill, one of the first and bitterest armed clashes +of the American Revolution, was fought in Boston's environs. Today there is +a monumental spire on Bunker Hill, visible throughout much of the city. +The willingness of the republican revolutionaries to take up arms and fire +on their oppressors has left a cultural legacy that two full centuries +have not effaced. Bunker Hill is still a potent center of American political +symbolism, and the Spirit of '76 is still a potent image for those who seek +to mold public opinion. + +Of course, not everyone who wraps himself in the flag is necessarily +a patriot. When I visited the spire in September 1991, it bore a huge, +badly-erased, spray-can grafitto around its bottom reading +"BRITS OUT--IRA PROVOS." Inside this hallowed edifice was +a glass-cased diorama of thousands of tiny toy soldiers, +rebels and redcoats, fighting and dying over the green hill, +the riverside marshes, the rebel trenchworks. Plaques indicated the +movement of troops, the shiftings of strategy. The Bunker Hill Monument +is occupied at its very center by the toy soldiers of a military +war-game simulation. + +The Boston metroplex is a place of great universities, +prominent among the Massachusetts Institute of Technology, +where the term "computer hacker" was first coined. The Hacker Crackdown +of 1990 might be interpreted as a political struggle among American cities: +traditional strongholds of longhair intellectual liberalism, +such as Boston, San Francisco, and Austin, versus the bare-knuckle +industrial pragmatism of Chicago and Phoenix (with Atlanta and New York +wrapped in internal struggle). + +The headquarters of the Electronic Frontier Foundation is on +155 Second Street in Cambridge, a Bostonian suburb north +of the River Charles. Second Street has weedy sidewalks of dented, +sagging brick and elderly cracked asphalt; large street-signs warn +"NO PARKING DURING DECLARED SNOW EMERGENCY." This is an old area +of modest manufacturing industries; the EFF is catecorner from the +Greene Rubber Company. EFF's building is two stories of red brick; +its large wooden windows feature gracefully arched tops and stone sills. + +The glass window beside the Second Street entrance bears three sheets +of neatly laser-printed paper, taped against the glass. They read: +ON Technology. EFF. KEI. + +"ON Technology" is Kapor's software company, which currently specializes +in "groupware" for the Apple Macintosh computer. "Groupware" is intended +to promote efficient social interaction among office-workers linked +by computers. ON Technology's most successful software products to date +are "Meeting Maker" and "Instant Update." + +"KEI" is Kapor Enterprises Inc., Kapor's personal holding company, +the commercial entity that formally controls his extensive investments +in other hardware and software corporations. + +"EFF" is a political action group--of a special sort. + +Inside, someone's bike has been chained to the handrails +of a modest flight of stairs. A wall of modish glass brick +separates this anteroom from the offices. Beyond the brick, +there's an alarm system mounted on the wall, a sleek, complex little +number that resembles a cross between a thermostat and a CD player. +Piled against the wall are box after box of a recent special issue +of Scientific American, "How to Work, Play, and Thrive in Cyberspace," +with extensive coverage of electronic networking techniques +and political issues, including an article by Kapor himself. +These boxes are addressed to Gerard Van der Leun, EFF's +Director of Communications, who will shortly mail those magazines +to every member of the EFF. + +The joint headquarters of EFF, KEI, and ON Technology, +which Kapor currently rents, is a modestly bustling place. +It's very much the same physical size as Steve Jackson's gaming company. +It's certainly a far cry from the gigantic gray steel-sided railway +shipping barn, on the Monsignor O'Brien Highway, that is owned +by Lotus Development Corporation. + +Lotus is, of course, the software giant that Mitchell Kapor founded +in the late 70s. The software program Kapor co-authored, +"Lotus 1-2-3," is still that company's most profitable product. +"Lotus 1-2-3" also bears a singular distinction in the +digital underground: it's probably the most pirated piece +of application software in world history. + +Kapor greets me cordially in his own office, down a hall. +Kapor, whose name is pronounced KAY-por, is in his early forties, +married and the father of two. He has a round face, high forehead, +straight nose, a slightly tousled mop of black hair peppered with gray. +His large brown eyes are wideset, reflective, one might almost say soulful. +He disdains ties, and commonly wears Hawaiian shirts and tropical prints, +not so much garish as simply cheerful and just that little bit anomalous. + +There is just the whiff of hacker brimstone about Mitch Kapor. +He may not have the hard-riding, hell-for-leather, guitar-strumming +charisma of his Wyoming colleague John Perry Barlow, but there's +something about the guy that still stops one short. He has the air +of the Eastern city dude in the bowler hat, the dreamy, +Longfellow-quoting poker shark who only HAPPENS to know +the exact mathematical odds against drawing to an inside straight. +Even among his computer-community colleagues, who are hardly known +for mental sluggishness, Kapor strikes one forcefully as a very +intelligent man. He speaks rapidly, with vigorous gestures, +his Boston accent sometimes slipping to the sharp nasal tang +of his youth in Long Island. + +Kapor, whose Kapor Family Foundation does much of his philanthropic work, +is a strong supporter of Boston's Computer Museum. Kapor's interest +in the history of his industry has brought him some remarkable curios, +such as the "byte" just outside his office door. This "byte"-- +eight digital bits--has been salvaged from the wreck of an +electronic computer of the pre-transistor age. It's a standing gunmetal +rack about the size of a small toaster-oven: with eight slots +of hand-soldered breadboarding featuring thumb-sized vacuum tubes. +If it fell off a table it could easily break your foot, +but it was state-of-the-art computation in the 1940s. +(It would take exactly 157,184 of these primordial toasters +to hold the first part of this book.) + +There's also a coiling, multicolored, scaly dragon that some +inspired techno-punk artist has cobbled up entirely out of transistors, +capacitors, and brightly plastic-coated wiring. + +Inside the office, Kapor excuses himself briefly to do a little +mouse-whizzing housekeeping on his personal Macintosh IIfx. +If its giant screen were an open window, an agile person +could climb through it without much trouble at all. +There's a coffee-cup at Kapor's elbow, a memento of his +recent trip to Eastern Europe, which has a black-and-white +stencilled photo and the legend CAPITALIST FOOLS TOUR. +It's Kapor, Barlow, and two California venture-capitalist luminaries +of their acquaintance, four windblown, grinning Baby Boomer +dudes in leather jackets, boots, denim, travel bags, +standing on airport tarmac somewhere behind the formerly Iron Curtain. +They look as if they're having the absolute time of their lives. + +Kapor is in a reminiscent mood. We talk a bit about his youth-- +high school days as a "math nerd," Saturdays attending Columbia University's +high-school science honors program, where he had his first experience +programming computers. IBM 1620s, in 1965 and '66. "I was very interested," +says Kapor, "and then I went off to college and got distracted by drugs sex +and rock and roll, like anybody with half a brain would have then!" +After college he was a progressive-rock DJ in Hartford, Connecticut, +for a couple of years. + +I ask him if he ever misses his rock and roll days--if he ever wished +he could go back to radio work. + +He shakes his head flatly. "I stopped thinking about going back +to be a DJ the day after Altamont." + +Kapor moved to Boston in 1974 and got a job programming mainframes in COBOL. +He hated it. He quit and became a teacher of transcendental meditation. +(It was Kapor's long flirtation with Eastern mysticism that gave the +world "Lotus.") + +In 1976 Kapor went to Switzerland, where the Transcendental Meditation +movement had rented a gigantic Victorian hotel in St-Moritz. It was +an all-male group--a hundred and twenty of them--determined upon +Enlightenment or Bust. Kapor had given the transcendant his best shot. +He was becoming disenchanted by "the nuttiness in the organization." +"They were teaching people to levitate," he says, staring at the floor. +His voice drops an octave, becomes flat. "THEY DON'T LEVITATE." + +Kapor chose Bust. He went back to the States and acquired a degree +in counselling psychology. He worked a while in a hospital, +couldn't stand that either. "My rep was," he says "a very bright kid +with a lot of potential who hasn't found himself. Almost thirty. +Sort of lost." + +Kapor was unemployed when he bought his first personal computer--an Apple II. +He sold his stereo to raise cash and drove to New Hampshire to avoid the +sales tax. + +"The day after I purchased it," Kapor tells me, "I was hanging out +in a computer store and I saw another guy, a man in his forties, +well-dressed guy, and eavesdropped on his conversation with the salesman. +He didn't know anything about computers. I'd had a year programming. +And I could program in BASIC. I'd taught myself. So I went up to him, +and I actually sold myself to him as a consultant." He pauses. +"I don't know where I got the nerve to do this. It was uncharacteristic. +I just said, `I think I can help you, I've been listening, +this is what you need to do and I think I can do it for you.' +And he took me on! He was my first client! I became a computer +consultant the first day after I bought the Apple II." + +Kapor had found his true vocation. He attracted more clients +for his consultant service, and started an Apple users' group. + +A friend of Kapor's, Eric Rosenfeld, a graduate student at MIT, +had a problem. He was doing a thesis on an arcane form of +financial statistics, but could not wedge himself into the crowded queue +for time on MIT's mainframes. (One might note at this point that if +Mr. Rosenfeld had dishonestly broken into the MIT mainframes, +Kapor himself might have never invented Lotus 1-2-3 and +the PC business might have been set back for years!) +Eric Rosenfeld did have an Apple II, however, +and he thought it might be possible to scale the problem down. +Kapor, as favor, wrote a program for him in BASIC that did the job. + +It then occurred to the two of them, out of the blue, +that it might be possible to SELL this program. +They marketed it themselves, in plastic baggies, +for about a hundred bucks a pop, mail order. +"This was a total cottage industry by a marginal consultant," +Kapor says proudly. "That's how I got started, honest to God." + +Rosenfeld, who later became a very prominent figure on Wall Street, +urged Kapor to go to MIT's business school for an MBA. +Kapor did seven months there, but never got his MBA. +He picked up some useful tools--mainly a firm grasp +of the principles of accounting--and, in his own words, +"learned to talk MBA." Then he dropped out and went to Silicon Valley. + +The inventors of VisiCalc, the Apple computer's premier business program, +had shown an interest in Mitch Kapor. Kapor worked diligently for them +for six months, got tired of California, and went back to Boston +where they had better bookstores. The VisiCalc group had made +the critical error of bringing in "professional management." +"That drove them into the ground," Kapor says. + +"Yeah, you don't hear a lot about VisiCalc these days," I muse. + +Kapor looks surprised. "Well, Lotus. . . we BOUGHT it." + +"Oh. You BOUGHT it?" + +"Yeah." + +"Sort of like the Bell System buying Western Union?" + +Kapor grins. "Yep! Yep! Yeah, exactly!" + +Mitch Kapor was not in full command of the destiny of himself +or his industry. The hottest software commodities of the early 1980s +were COMPUTER GAMES--the Atari seemed destined to enter every teenage home +in America. Kapor got into business software simply because he didn't have +any particular feeling for computer games. But he was supremely fast +on his feet, open to new ideas and inclined to trust his instincts. +And his instincts were good. He chose good people to deal with-- +gifted programmer Jonathan Sachs (the co-author of Lotus 1-2-3). +Financial wizard Eric Rosenfeld, canny Wall Street analyst +and venture capitalist Ben Rosen. Kapor was the founder and CEO of Lotus, +one of the most spectacularly successful business ventures of the +later twentieth century. + +He is now an extremely wealthy man. I ask him if he actually +knows how much money he has. + +"Yeah," he says. "Within a percent or two." + +How much does he actually have, then? + +He shakes his head. "A lot. A lot. Not something I talk about. +Issues of money and class are things that cut pretty close to the bone." + +I don't pry. It's beside the point. One might presume, impolitely, +that Kapor has at least forty million--that's what he got the year +he left Lotus. People who ought to know claim Kapor has about +a hundred and fifty million, give or take a market swing +in his stock holdings. If Kapor had stuck with Lotus, +as his colleague friend and rival Bill Gates has stuck +with his own software start-up, Microsoft, then Kapor +would likely have much the same fortune Gates has-- +somewhere in the neighborhood of three billion, +give or take a few hundred million. Mitch Kapor +has all the money he wants. Money has lost whatever charm +it ever held for him--probably not much in the first place. +When Lotus became too uptight, too bureaucratic, too far +from the true sources of his own satisfaction, Kapor walked. +He simply severed all connections with the company and went out the door. +It stunned everyone--except those who knew him best. + +Kapor has not had to strain his resources to wreak a thorough +transformation in cyberspace politics. In its first year, +EFF's budget was about a quarter of a million dollars. +Kapor is running EFF out of his pocket change. + +Kapor takes pains to tell me that he does not consider himself +a civil libertarian per se. He has spent quite some time +with true-blue civil libertarians lately, and there's a +political-correctness to them that bugs him. They seem +to him to spend entirely too much time in legal nitpicking +and not enough vigorously exercising civil rights in the +everyday real world. + +Kapor is an entrepreneur. Like all hackers, he prefers his involvements +direct, personal, and hands-on. "The fact that EFF has a node on the +Internet is a great thing. We're a publisher. We're a distributor +of information." Among the items the eff.org Internet node carries +is back issues of Phrack. They had an internal debate about that in EFF, +and finally decided to take the plunge. They might carry other +digital underground publications--but if they do, he says, +"we'll certainly carry Donn Parker, and anything Gail Thackeray +wants to put up. We'll turn it into a public library, that has +the whole spectrum of use. Evolve in the direction of people making up +their own minds." He grins. "We'll try to label all the editorials." + +Kapor is determined to tackle the technicalities of the Internet +in the service of the public interest. "The problem with being a node +on the Net today is that you've got to have a captive technical specialist. +We have Chris Davis around, for the care and feeding of the balky beast! +We couldn't do it ourselves!" + +He pauses. "So one direction in which technology has to evolve +is much more standardized units, that a non-technical person +can feel comfortable with. It's the same shift as from minicomputers to PCs. +I can see a future in which any person can have a Node on the Net. +Any person can be a publisher. It's better than the media we now have. +It's possible. We're working actively." + +Kapor is in his element now, fluent, thoroughly in command in his material. +"You go tell a hardware Internet hacker that everyone should have a node +on the Net," he says, "and the first thing they're going to say is, +`IP doesn't scale!'" ("IP" is the interface protocol for the Internet. +As it currently exists, the IP software is simply not capable of +indefinite expansion; it will run out of usable addresses, it will saturate.) +"The answer," Kapor says, "is: evolve the protocol! Get the smart people +together and figure out what to do. Do we add ID? Do we add new protocol? +Don't just say, WE CAN'T DO IT." + +Getting smart people together to figure out what to do is a skill +at which Kapor clearly excels. I counter that people on the Internet +rather enjoy their elite technical status, and don't seem particularly +anxious to democratize the Net. + +Kapor agrees, with a show of scorn. "I tell them that this is the snobbery +of the people on the Mayflower looking down their noses at the people +who came over ON THE SECOND BOAT! Just because they got here a year, +or five years, or ten years before everybody else, that doesn't give +them ownership of cyberspace! By what right?" + +I remark that the telcos are an electronic network, too, +and they seem to guard their specialized knowledge pretty closely. + +Kapor ripostes that the telcos and the Internet are entirely +different animals. "The Internet is an open system, +everything is published, everything gets argued about, +basically by anybody who can get in. Mostly, it's exclusive +and elitist just because it's so difficult. Let's make it easier to use." + +On the other hand, he allows with a swift change of emphasis, +the so-called elitists do have a point as well. "Before people start coming in, +who are new, who want to make suggestions, and criticize the Net as +`all screwed up'. . . . They should at least take the time to understand +the culture on its own terms. It has its own history--show some respect +for it. I'm a conservative, to that extent." + +The Internet is Kapor's paradigm for the future of telecommunications. +The Internet is decentralized, non-hierarchical, almost anarchic. +There are no bosses, no chain of command, no secret data. +If each node obeys the general interface standards, +there's simply no need for any central network authority. + +Wouldn't that spell the doom of AT&T as an institution? I ask. + +That prospect doesn't faze Kapor for a moment. "Their big advantage, +that they have now, is that they have all of the wiring. +But two things are happening. Anyone with right-of-way +is putting down fiber--Southern Pacific Railroad, +people like that--there's enormous `dark fiber' laid in." +("Dark Fiber" is fiber-optic cable, whose enormous capacity +so exceeds the demands of current usage that much of the +fiber still has no light-signals on it--it's still `dark,' +awaiting future use.) + +"The other thing that's happening is the local-loop stuff +is going to go wireless. Everyone from Bellcore to the cable TV +companies to AT&T wants to put in these things called +`personal communication systems.' So you could have local competition-- +you could have multiplicity of people, a bunch of neighborhoods, +sticking stuff up on poles. And a bunch of other people laying in dark fiber. +So what happens to the telephone companies? There's enormous pressure +on them from both sides. + +"The more I look at this, the more I believe that in a post-industrial, +digital world, the idea of regulated monopolies is bad. People will +look back on it and say that in the 19th and 20th centuries +the idea of public utilities was an okay compromise. +You needed one set of wires in the ground. It was too economically +inefficient, otherwise. And that meant one entity running it. +But now, with pieces being wireless--the connections are going +to be via high-level interfaces, not via wires. I mean, ULTIMATELY +there are going to be wires--but the wires are just a commodity. +Fiber, wireless. You no longer NEED a utility." + +Water utilities? Gas utilities? + +Of course we still need those, he agrees. "But when what you're moving +is information, instead of physical substances, then you can play by +a different set of rules. We're evolving those rules now! +Hopefully you can have a much more decentralized system, +and one in which there's more competition in the marketplace. + +"The role of government will be to make sure that nobody cheats. +The proverbial `level playing field.' A policy that prevents monopolization. +It should result in better service, lower prices, more choices, +and local empowerment." He smiles. "I'm very big on local empowerment." + +Kapor is a man with a vision. It's a very novel vision which he +and his allies are working out in considerable detail and with great energy. +Dark, cynical, morbid cyberpunk that I am, I cannot avoid considering +some of the darker implications of "decentralized, nonhierarchical, +locally empowered" networking. + +I remark that some pundits have suggested that electronic networking--faxes, +phones, small-scale photocopiers--played a strong role in dissolving +the power of centralized communism and causing the collapse of the Warsaw Pact. + +Socialism is totally discredited, says Kapor, fresh back from +the Eastern Bloc. The idea that faxes did it, all by themselves, +is rather wishful thinking. + +Has it occurred to him that electronic networking might corrode +America's industrial and political infrastructure to the point +where the whole thing becomes untenable, unworkable--and the old order +just collapses headlong, like in Eastern Europe? + +"No," Kapor says flatly. "I think that's extraordinarily unlikely. +In part, because ten or fifteen years ago, I had similar hopes +about personal computers--which utterly failed to materialize." +He grins wryly, then his eyes narrow. "I'm VERY opposed to techno-utopias. +Every time I see one, I either run away, or try to kill it." + +It dawns on me then that Mitch Kapor is not trying to +make the world safe for democracy. He certainly is not +trying to make it safe for anarchists or utopians-- +least of all for computer intruders or electronic rip-off artists. +What he really hopes to do is make the world safe for +future Mitch Kapors. This world of decentralized, small-scale nodes, +with instant global access for the best and brightest, +would be a perfect milieu for the shoestring attic capitalism +that made Mitch Kapor what he is today. + +Kapor is a very bright man. He has a rare combination +of visionary intensity with a strong practical streak. +The Board of the EFF: John Barlow, Jerry Berman of the ACLU, +Stewart Brand, John Gilmore, Steve Wozniak, and Esther Dyson, +the doyenne of East-West computer entrepreneurism--share his gift, +his vision, and his formidable networking talents. +They are people of the 1960s, winnowed-out by its turbulence +and rewarded with wealth and influence. They are some of the best +and the brightest that the electronic community has to offer. +But can they do it, in the real world? Or are they only dreaming? +They are so few. And there is so much against them. + +I leave Kapor and his networking employees struggling cheerfully +with the promising intricacies of their newly installed Macintosh +System 7 software. The next day is Saturday. EFF is closed. +I pay a few visits to points of interest downtown. + +One of them is the birthplace of the telephone. + +It's marked by a bronze plaque in a plinth of black-and-white speckled granite. It sits in the +plaza of the John F. Kennedy Federal Building, the very place where Kapor was +once fingerprinted by the FBI. + +The plaque has a bas-relief picture of Bell's original telephone. +"BIRTHPLACE OF THE TELEPHONE," it reads. "Here, on June 2, 1875, +Alexander Graham Bell and Thomas A. Watson first transmitted sound over wires. + +"This successful experiment was completed in a fifth floor garret +at what was then 109 Court Street and marked the beginning of +world-wide telephone service." + +109 Court Street is long gone. Within sight of Bell's plaque, +across a street, is one of the central offices of NYNEX, +the local Bell RBOC, on 6 Bowdoin Square. + +I cross the street and circle the telco building, slowly, +hands in my jacket pockets. It's a bright, windy, New England +autumn day. The central office is a handsome 1940s-era megalith +in late Art Deco, eight stories high. + +Parked outside the back is a power-generation truck. +The generator strikes me as rather anomalous. Don't they +already have their own generators in this eight-story monster? +Then the suspicion strikes me that NYNEX must have heard +of the September 17 AT&T power-outage which crashed New York City. +Belt-and-suspenders, this generator. Very telco. + +Over the glass doors of the front entrance is a handsome bronze +bas-relief of Art Deco vines, sunflowers, and birds, entwining +the Bell logo and the legend NEW ENGLAND TELEPHONE AND TELEGRAPH COMPANY +--an entity which no longer officially exists. + +The doors are locked securely. I peer through the shadowed glass. +Inside is an official poster reading: + + +"New England Telephone a NYNEX Company + +ATTENTION + +"All persons while on New England Telephone +Company premises are required to visibly wear their +identification cards (C.C.P. Section 2, Page 1). + +"Visitors, vendors, contractors, and all others are +required to visibly wear a daily pass. + +"Thank you. + +Kevin C. Stanton. +Building Security Coordinator." + + +Outside, around the corner, is a pull-down ribbed metal security door, +a locked delivery entrance. Some passing stranger has grafitti-tagged +this door, with a single word in red spray-painted cursive: + +Fury + +# + +My book on the Hacker Crackdown is almost over now. +I have deliberately saved the best for last. + +In February 1991, I attended the CPSR Public Policy Roundtable, +in Washington, DC. CPSR, Computer Professionals for Social Responsibility, +was a sister organization of EFF, or perhaps its aunt, being older +and perhaps somewhat wiser in the ways of the world of politics. + +Computer Professionals for Social Responsibility began in 1981 +in Palo Alto, as an informal discussion group of Californian +computer scientists and technicians, united by nothing more +than an electronic mailing list. This typical high-tech +ad-hocracy received the dignity of its own acronym in 1982, +and was formally incorporated in 1983. + +CPSR lobbied government and public alike with an educational +outreach effort, sternly warning against any foolish +and unthinking trust in complex computer systems. +CPSR insisted that mere computers should never be +considered a magic panacea for humanity's social, +ethical or political problems. CPSR members were especially +troubled about the stability, safety, and dependability +of military computer systems, and very especially troubled +by those systems controlling nuclear arsenals. CPSR was +best-known for its persistent and well-publicized attacks on the +scientific credibility of the Strategic Defense Initiative ("Star Wars"). + +In 1990, CPSR was the nation's veteran cyber-political activist group, +with over two thousand members in twenty- one local chapters across the US. +It was especially active in Boston, Silicon Valley, and Washington DC, +where its Washington office sponsored the Public Policy Roundtable. + +The Roundtable, however, had been funded by EFF, which had passed CPSR +an extensive grant for operations. This was the first large-scale, +official meeting of what was to become the electronic civil +libertarian community. + +Sixty people attended, myself included--in this instance, not so much +as a journalist as a cyberpunk author. Many of the luminaries +of the field took part: Kapor and Godwin as a matter of course. +Richard Civille and Marc Rotenberg of CPSR. Jerry Berman of the ACLU. +John Quarterman, author of The Matrix. Steven Levy, author of Hackers. +George Perry and Sandy Weiss of Prodigy Services, there to network +about the civil-liberties troubles their young commercial +network was experiencing. Dr. Dorothy Denning. Cliff Figallo, +manager of the Well. Steve Jackson was there, having finally +found his ideal target audience, and so was Craig Neidorf, +"Knight Lightning" himself, with his attorney, Sheldon Zenner. +Katie Hafner, science journalist, and co-author of Cyberpunk: +Outlaws and Hackers on the Computer Frontier. Dave Farber, +ARPAnet pioneer and fabled Internet guru. Janlori Goldman +of the ACLU's Project on Privacy and Technology. John Nagle +of Autodesk and the Well. Don Goldberg of the House Judiciary Committee. +Tom Guidoboni, the defense attorney in the Internet Worm case. +Lance Hoffman, computer-science professor at The George Washington +University. Eli Noam of Columbia. And a host of others no less distinguished. + +Senator Patrick Leahy delivered the keynote address, +expressing his determination to keep ahead of the curve +on the issue of electronic free speech. The address was +well-received, and the sense of excitement was palpable. +Every panel discussion was interesting--some were entirely +compelling. People networked with an almost frantic interest. + +I myself had a most interesting and cordial lunch discussion with +Noel and Jeanne Gayler, Admiral Gayler being a former director +of the National Security Agency. As this was the first known encounter +between an actual no-kidding cyberpunk and a chief executive of +America's largest and best-financed electronic espionage apparat, +there was naturally a bit of eyebrow-raising on both sides. + +Unfortunately, our discussion was off-the-record. In fact +all the discussions at the CPSR were officially off-the-record, +the idea being to do some serious networking in an atmosphere +of complete frankness, rather than to stage a media circus. + +In any case, CPSR Roundtable, though interesting and intensely valuable, +was as nothing compared to the truly mind-boggling event that transpired +a mere month later. + +# + +"Computers, Freedom and Privacy." Four hundred people from +every conceivable corner of America's electronic community. +As a science fiction writer, I have been to some weird gigs in my day, +but this thing is truly BEYOND THE PALE. Even "Cyberthon," +Point Foundation's "Woodstock of Cyberspace" where Bay Area +psychedelia collided headlong with the emergent world +of computerized virtual reality, was like a Kiwanis Club gig +compared to this astonishing do. + +The "electronic community" had reached an apogee. +Almost every principal in this book is in attendance. +Civil Libertarians. Computer Cops. The Digital Underground. +Even a few discreet telco people. Colorcoded dots +for lapel tags are distributed. Free Expression issues. +Law Enforcement. Computer Security. Privacy. Journalists. +Lawyers. Educators. Librarians. Programmers. +Stylish punk-black dots for the hackers and phone phreaks. +Almost everyone here seems to wear eight or nine dots, +to have six or seven professional hats. + +It is a community. Something like Lebanon perhaps, +but a digital nation. People who had feuded all year +in the national press, people who entertained the deepest +suspicions of one another's motives and ethics, are now +in each others' laps. "Computers, Freedom and Privacy" +had every reason in the world to turn ugly, and yet except +for small irruptions of puzzling nonsense from the +convention's token lunatic, a surprising bonhomie reigned. +CFP was like a wedding-party in which two lovers, +unstable bride and charlatan groom, tie the knot +in a clearly disastrous matrimony. + +It is clear to both families--even to neighbors and random guests-- +that this is not a workable relationship, and yet the young couple's +desperate attraction can brook no further delay. They simply cannot +help themselves. Crockery will fly, shrieks from their newlywed home +will wake the city block, divorce waits in the wings like a vulture +over the Kalahari, and yet this is a wedding, and there is going +to be a child from it. Tragedies end in death; comedies in marriage. +The Hacker Crackdown is ending in marriage. And there will be a child. + +From the beginning, anomalies reign. John Perry Barlow, +cyberspace ranger, is here. His color photo in +The New York Times Magazine, Barlow scowling +in a grim Wyoming snowscape, with long black coat, +dark hat, a Macintosh SE30 propped on a fencepost +and an awesome frontier rifle tucked under one arm, +will be the single most striking visual image +of the Hacker Crackdown. And he is CFP's guest of honor-- +along with Gail Thackeray of the FCIC! What on earth do +they expect these dual guests to do with each other? Waltz? + +Barlow delivers the first address. Uncharacteristically, +he is hoarse--the sheer volume of roadwork has worn him down. +He speaks briefly, congenially, in a plea for conciliation, +and takes his leave to a storm of applause. + +Then Gail Thackeray takes the stage. She's visibly nervous. +She's been on the Well a lot lately. Reading those Barlow posts. +Following Barlow is a challenge to anyone. In honor of the famous +lyricist for the Grateful Dead, she announces reedily, she is going to read-- +A POEM. A poem she has composed herself. + +It's an awful poem, doggerel in the rollicking meter of Robert W. Service's +The Cremation of Sam McGee, but it is in fact, a poem. It's the Ballad +of the Electronic Frontier! A poem about the Hacker Crackdown and the +sheer unlikelihood of CFP. It's full of in-jokes. The score or so cops +in the audience, who are sitting together in a nervous claque, +are absolutely cracking-up. Gail's poem is the funniest goddamn thing +they've ever heard. The hackers and civil-libs, who had this woman figured +for Ilsa She-Wolf of the SS, are staring with their jaws hanging loosely. +Never in the wildest reaches of their imagination had they figured +Gail Thackeray was capable of such a totally off-the-wall move. +You can see them punching their mental CONTROL-RESET buttons. +Jesus! This woman's a hacker weirdo! She's JUST LIKE US! +God, this changes everything! + +Al Bayse, computer technician for the FBI, had been the only cop +at the CPSR Roundtable, dragged there with his arm bent by +Dorothy Denning. He was guarded and tightlipped at CPSR Roundtable; +a "lion thrown to the Christians." + +At CFP, backed by a claque of cops, Bayse suddenly waxes eloquent +and even droll, describing the FBI's "NCIC 2000", a gigantic digital catalog +of criminal records, as if he has suddenly become some weird hybrid +of George Orwell and George Gobel. Tentatively, he makes an arcane +joke about statistical analysis. At least a third of the crowd laughs aloud. + +"They didn't laugh at that at my last speech," Bayse observes. +He had been addressing cops--STRAIGHT cops, not computer people. +It had been a worthy meeting, useful one supposes, but nothing like THIS. +There has never been ANYTHING like this. Without any prodding, +without any preparation, people in the audience simply begin to ask questions. +Longhairs, freaky people, mathematicians. Bayse is answering, politely, +frankly, fully, like a man walking on air. The ballroom's atmosphere +crackles with surreality. A female lawyer behind me breaks into a sweat +and a hot waft of surprisingly potent and musky perfume flows off +her pulse-points. + +People are giddy with laughter. People are interested, +fascinated, their eyes so wide and dark that they seem eroticized. +Unlikely daisy-chains form in the halls, around the bar, on the escalators: +cops with hackers, civil rights with FBI, Secret Service with phone phreaks. + +Gail Thackeray is at her crispest in a white wool sweater with a +tiny Secret Service logo. "I found Phiber Optik at the payphones, +and when he saw my sweater, he turned into a PILLAR OF SALT!" she chortles. + +Phiber discusses his case at much length with his arresting officer, +Don Delaney of the New York State Police. After an hour's chat, +the two of them look ready to begin singing "Auld Lang Syne." +Phiber finally finds the courage to get his worst complaint off his chest. +It isn't so much the arrest. It was the CHARGE. Pirating service +off 900 numbers. I'm a PROGRAMMER, Phiber insists. This lame charge +is going to hurt my reputation. It would have been cool to be busted +for something happening, like Section 1030 computer intrusion. +Maybe some kind of crime that's scarcely been invented yet. +Not lousy phone fraud. Phooey. + +Delaney seems regretful. He had a mountain of possible criminal charges +against Phiber Optik. The kid's gonna plead guilty anyway. He's a +first timer, they always plead. Coulda charged the kid with most anything, +and gotten the same result in the end. Delaney seems genuinely sorry +not to have gratified Phiber in this harmless fashion. Too late now. +Phiber's pled already. All water under the bridge. Whaddya gonna do? + +Delaney's got a good grasp on the hacker mentality. +He held a press conference after he busted a bunch of +Masters of Deception kids. Some journo had asked him: +"Would you describe these people as GENIUSES?" +Delaney's deadpan answer, perfect: "No, I would describe +these people as DEFENDANTS." Delaney busts a kid for +hacking codes with repeated random dialling. Tells the +press that NYNEX can track this stuff in no time flat nowadays, +and a kid has to be STUPID to do something so easy to catch. +Dead on again: hackers don't mind being thought of as Genghis Khan +by the straights, but if there's anything that really gets 'em +where they live, it's being called DUMB. + +Won't be as much fun for Phiber next time around. +As a second offender he's gonna see prison. +Hackers break the law. They're not geniuses, either. +They're gonna be defendants. And yet, Delaney muses over +a drink in the hotel bar, he has found it impossible to treat +them as common criminals. Delaney knows criminals. These kids, +by comparison, are clueless--there is just no crook vibe off of them, +they don't smell right, they're just not BAD. + +Delaney has seen a lot of action. He did Vietnam. +He's been shot at, he has shot people. He's a homicide +cop from New York. He has the appearance of a man who +has not only seen the shit hit the fan but has seen it splattered +across whole city blocks and left to ferment for years. +This guy has been around. + +He listens to Steve Jackson tell his story. The dreamy +game strategist has been dealt a bad hand. He has played +it for all he is worth. Under his nerdish SF-fan exterior +is a core of iron. Friends of his say Steve Jackson believes +in the rules, believes in fair play. He will never compromise +his principles, never give up. "Steve," Delaney says to +Steve Jackson, "they had some balls, whoever busted you. +You're all right!" Jackson, stunned, falls silent and +actually blushes with pleasure. + +Neidorf has grown up a lot in the past year. The kid is +a quick study, you gotta give him that. Dressed by his mom, +the fashion manager for a national clothing chain, +Missouri college techie-frat Craig Neidorf out-dappers +everyone at this gig but the toniest East Coast lawyers. +The iron jaws of prison clanged shut without him and now +law school beckons for Neidorf. He looks like a larval Congressman. + +Not a "hacker," our Mr. Neidorf. He's not interested +in computer science. Why should he be? He's not +interested in writing C code the rest of his life, +and besides, he's seen where the chips fall. +To the world of computer science he and Phrack +were just a curiosity. But to the world of law. . . . +The kid has learned where the bodies are buried. +He carries his notebook of press clippings wherever he goes. + +Phiber Optik makes fun of Neidorf for a Midwestern geek, +for believing that "Acid Phreak" does acid and listens to acid rock. +Hell no. Acid's never done ACID! Acid's into ACID HOUSE MUSIC. +Jesus. The very idea of doing LSD. Our PARENTS did LSD, ya clown. + +Thackeray suddenly turns upon Craig Neidorf the full lighthouse +glare of her attention and begins a determined half-hour attempt +to WIN THE BOY OVER. The Joan of Arc of Computer Crime is +GIVING CAREER ADVICE TO KNIGHT LIGHTNING! "Your experience +would be very valuable--a real asset," she tells him with +unmistakeable sixty-thousand-watt sincerity. Neidorf is fascinated. +He listens with unfeigned attention. He's nodding and saying yes ma'am. +Yes, Craig, you too can forget all about money and enter the glamorous +and horribly underpaid world of PROSECUTING COMPUTER CRIME! +You can put your former friends in prison--ooops. . . . + +You cannot go on dueling at modem's length indefinitely. +You cannot beat one another senseless with rolled-up press-clippings. +Sooner or later you have to come directly to grips. +And yet the very act of assembling here has changed +the entire situation drastically. John Quarterman, +author of The Matrix, explains the Internet at his symposium. +It is the largest news network in the world, it is growing +by leaps and bounds, and yet you cannot measure Internet because +you cannot stop it in place. It cannot stop, because there +is no one anywhere in the world with the authority to stop Internet. +It changes, yes, it grows, it embeds itself across the post-industrial, +postmodern world and it generates community wherever it +touches, and it is doing this all by itself. + +Phiber is different. A very fin de siecle kid, Phiber Optik. +Barlow says he looks like an Edwardian dandy. He does rather. +Shaven neck, the sides of his skull cropped hip-hop close, +unruly tangle of black hair on top that looks pomaded, +he stays up till four a.m. and misses all the sessions, +then hangs out in payphone booths with his acoustic coupler +gutsily CRACKING SYSTEMS RIGHT IN THE MIDST OF THE HEAVIEST +LAW ENFORCEMENT DUDES IN THE U.S., or at least PRETENDING to. . . . +Unlike "Frank Drake." Drake, who wrote Dorothy Denning out +of nowhere, and asked for an interview for his cheapo +cyberpunk fanzine, and then started grilling her on her ethics. +She was squirmin', too. . . . Drake, scarecrow-tall with his +floppy blond mohawk, rotting tennis shoes and black leather jacket +lettered ILLUMINATI in red, gives off an unmistakeable air +of the bohemian literatus. Drake is the kind of guy +who reads British industrial design magazines and appreciates +William Gibson because the quality of the prose is so tasty. +Drake could never touch a phone or a keyboard again, +and he'd still have the nose-ring and the blurry photocopied +fanzines and the sampled industrial music. He's a radical punk +with a desktop-publishing rig and an Internet address. +Standing next to Drake, the diminutive Phiber looks like he's +been physically coagulated out of phone-lines. Born to phreak. + +Dorothy Denning approaches Phiber suddenly. The two of them +are about the same height and body-build. Denning's blue eyes +flash behind the round window-frames of her glasses. +"Why did you say I was `quaint?'" she asks Phiber, quaintly. + +It's a perfect description but Phiber is nonplussed. . . +"Well, I uh, you know. . . ." + +"I also think you're quaint, Dorothy," I say, novelist to the rescue, +the journo gift of gab. . . . She is neat and dapper and yet there's +an arcane quality to her, something like a Pilgrim Maiden behind +leaded glass; if she were six inches high Dorothy Denning would look +great inside a china cabinet. . .The Cryptographeress. . . +The Cryptographrix. . .whatever. . . . Weirdly, Peter Denning looks +just like his wife, you could pick this gentleman out of a thousand guys +as the soulmate of Dorothy Denning. Wearing tailored slacks, +a spotless fuzzy varsity sweater, and a neatly knotted academician's tie. . . . +This fineboned, exquisitely polite, utterly civilized and hyperintelligent +couple seem to have emerged from some cleaner and finer parallel universe, +where humanity exists to do the Brain Teasers column in Scientific American. +Why does this Nice Lady hang out with these unsavory characters? + +Because the time has come for it, that's why. +Because she's the best there is at what she does. + +Donn Parker is here, the Great Bald Eagle of Computer Crime. . . . +With his bald dome, great height, and enormous Lincoln-like hands, +the great visionary pioneer of the field plows through the lesser mortals +like an icebreaker. . . . His eyes are fixed on the future with the +rigidity of a bronze statue. . . . Eventually, he tells his audience, +all business crime will be computer crime, because businesses will do +everything through computers. "Computer crime" as a category will vanish. + +In the meantime, passing fads will flourish and fail and evaporate. . . . +Parker's commanding, resonant voice is sphinxlike, everything is viewed +from some eldritch valley of deep historical abstraction. . . . +Yes, they've come and they've gone, these passing flaps in the world +of digital computation. . . . The radio-frequency emanation scandal. . . +KGB and MI5 and CIA do it every day, it's easy, but nobody else ever has. . . . +The salami-slice fraud, mostly mythical. . . . "Crimoids," he calls them. . . . +Computer viruses are the current crimoid champ, a lot less dangerous than +most people let on, but the novelty is fading and there's a crimoid vacuum at +the moment, the press is visibly hungering for something more outrageous. . . . +The Great Man shares with us a few speculations on the coming crimoids. . . . +Desktop Forgery! Wow. . . . Computers stolen just for the sake of the +information within them--data-napping! Happened in Britain a while ago, +could be the coming thing. . . . Phantom nodes in the Internet! + +Parker handles his overhead projector sheets with an ecclesiastical air. . . . +He wears a grey double-breasted suit, a light blue shirt, and a +very quiet tie of understated maroon and blue paisley. . . . +Aphorisms emerge from him with slow, leaden emphasis. . . . +There is no such thing as an adequately secure computer +when one faces a sufficiently powerful adversary. . . . +Deterrence is the most socially useful aspect of security. . . . +People are the primary weakness in all information systems. . . . +The entire baseline of computer security must be shifted upward. . . . +Don't ever violate your security by publicly describing +your security measures. . . . + +People in the audience are beginning to squirm, and yet +there is something about the elemental purity of this guy's +philosophy that compels uneasy respect. . . . Parker sounds +like the only sane guy left in the lifeboat, sometimes. +The guy who can prove rigorously, from deep moral principles, +that Harvey there, the one with the broken leg and the checkered past, +is the one who has to be, err. . .that is, Mr. Harvey is best placed +to make the necessary sacrifice for the security and indeed +the very survival of the rest of this lifeboat's crew. . . . +Computer security, Parker informs us mournfully, is a +nasty topic, and we wish we didn't have to have it. . . . +The security expert, armed with method and logic, must think--imagine-- +everything that the adversary might do before the adversary might +actually do it. It is as if the criminal's dark brain were an +extensive subprogram within the shining cranium of Donn Parker. +He is a Holmes whose Moriarty does not quite yet exist +and so must be perfectly simulated. + +CFP is a stellar gathering, with the giddiness of a wedding. +It is a happy time, a happy ending, they know their world +is changing forever tonight, and they're proud to have been there +to see it happen, to talk, to think, to help. + +And yet as night falls, a certain elegiac quality manifests itself, +as the crowd gathers beneath the chandeliers with their wineglasses +and dessert plates. Something is ending here, gone forever, +and it takes a while to pinpoint it. + +It is the End of the Amateurs. + + + + + + + + + +End of the Project Gutenberg EBook of Hacker Crackdown, by Bruce Sterling + +*** END OF THIS PROJECT GUTENBERG EBOOK HACKER CRACKDOWN *** + +***** This file should be named 101.txt or 101.zip ***** +This and all associated files of various formats will be found in: + http://www.gutenberg.org/1/0/101/ + + + +Updated editions will replace the previous one--the old editions will be +renamed. + +Creating the works from public domain print editions means that no one +owns a United States copyright in these works, so the Foundation (and +you!) can copy and distribute it in the United States without permission +and without paying copyright royalties. Special rules, set forth in the +General Terms of Use part of this license, apply to copying and +distributing Project Gutenberg-tm electronic works to protect the +PROJECT GUTENBERG-tm concept and trademark. Project Gutenberg is a +registered trademark, and may not be used if you charge for the eBooks, +unless you receive specific permission. If you do not charge anything +for copies of this eBook, complying with the rules is very easy. You may +use this eBook for nearly any purpose such as creation of derivative +works, reports, performances and research. They may be modified and +printed and given away--you may do practically ANYTHING with public +domain eBooks. Redistribution is subject to the trademark license, +especially commercial redistribution. + + + +*** START: FULL LICENSE *** + +THE FULL PROJECT GUTENBERG LICENSE +PLEASE READ THIS BEFORE YOU DISTRIBUTE OR USE THIS WORK + +To protect the Project Gutenberg-tm mission of promoting the free +distribution of electronic works, by using or distributing this work +(or any other work associated in any way with the phrase "Project +Gutenberg"), you agree to comply with all the terms of the Full Project +Gutenberg-tm License (available with this file or online at +http://www.gutenberg.org/license). + + +Section 1. General Terms of Use and Redistributing Project Gutenberg-tm +electronic works + +1.A. By reading or using any part of this Project Gutenberg-tm +electronic work, you indicate that you have read, understand, agree to +and accept all the terms of this license and intellectual property +(trademark/copyright) agreement. If you do not agree to abide by all +the terms of this agreement, you must cease using and return or destroy +all copies of Project Gutenberg-tm electronic works in your possession. +If you paid a fee for obtaining a copy of or access to a Project +Gutenberg-tm electronic work and you do not agree to be bound by the +terms of this agreement, you may obtain a refund from the person or +entity to whom you paid the fee as set forth in paragraph 1.E.8. + +1.B. "Project Gutenberg" is a registered trademark. It may only be +used on or associated in any way with an electronic work by people who +agree to be bound by the terms of this agreement. There are a few +things that you can do with most Project Gutenberg-tm electronic works +even without complying with the full terms of this agreement. See +paragraph 1.C below. There are a lot of things you can do with Project +Gutenberg-tm electronic works if you follow the terms of this agreement +and help preserve free future access to Project Gutenberg-tm electronic +works. See paragraph 1.E below. + +1.C. The Project Gutenberg Literary Archive Foundation ("the Foundation" +or PGLAF), owns a compilation copyright in the collection of Project +Gutenberg-tm electronic works. Nearly all the individual works in the +collection are in the public domain in the United States. If an +individual work is in the public domain in the United States and you are +located in the United States, we do not claim a right to prevent you from +copying, distributing, performing, displaying or creating derivative +works based on the work as long as all references to Project Gutenberg +are removed. Of course, we hope that you will support the Project +Gutenberg-tm mission of promoting free access to electronic works by +freely sharing Project Gutenberg-tm works in compliance with the terms of +this agreement for keeping the Project Gutenberg-tm name associated with +the work. You can easily comply with the terms of this agreement by +keeping this work in the same format with its attached full Project +Gutenberg-tm License when you share it without charge with others. +This particular work is one of the few copyrighted individual works +included with the permission of the copyright holder. Information on +the copyright owner for this particular work and the terms of use +imposed by the copyright holder on this work are set forth at the +beginning of this work. + +1.D. The copyright laws of the place where you are located also govern +what you can do with this work. Copyright laws in most countries are in +a constant state of change. If you are outside the United States, check +the laws of your country in addition to the terms of this agreement +before downloading, copying, displaying, performing, distributing or +creating derivative works based on this work or any other Project +Gutenberg-tm work. The Foundation makes no representations concerning +the copyright status of any work in any country outside the United +States. + +1.E. Unless you have removed all references to Project Gutenberg: + +1.E.1. The following sentence, with active links to, or other immediate +access to, the full Project Gutenberg-tm License must appear prominently +whenever any copy of a Project Gutenberg-tm work (any work on which the +phrase "Project Gutenberg" appears, or with which the phrase "Project +Gutenberg" is associated) is accessed, displayed, performed, viewed, +copied or distributed: + +This eBook is for the use of anyone anywhere at no cost and with +almost no restrictions whatsoever. You may copy it, give it away or +re-use it under the terms of the Project Gutenberg License included +with this eBook or online at www.gutenberg.org + +1.E.2. If an individual Project Gutenberg-tm electronic work is derived +from the public domain (does not contain a notice indicating that it is +posted with permission of the copyright holder), the work can be copied +and distributed to anyone in the United States without paying any fees +or charges. If you are redistributing or providing access to a work +with the phrase "Project Gutenberg" associated with or appearing on the +work, you must comply either with the requirements of paragraphs 1.E.1 +through 1.E.7 or obtain permission for the use of the work and the +Project Gutenberg-tm trademark as set forth in paragraphs 1.E.8 or +1.E.9. + +1.E.3. If an individual Project Gutenberg-tm electronic work is posted +with the permission of the copyright holder, your use and distribution +must comply with both paragraphs 1.E.1 through 1.E.7 and any additional +terms imposed by the copyright holder. Additional terms will be linked +to the Project Gutenberg-tm License for all works posted with the +permission of the copyright holder found at the beginning of this work. + +1.E.4. Do not unlink or detach or remove the full Project Gutenberg-tm +License terms from this work, or any files containing a part of this +work or any other work associated with Project Gutenberg-tm. + +1.E.5. Do not copy, display, perform, distribute or redistribute this +electronic work, or any part of this electronic work, without +prominently displaying the sentence set forth in paragraph 1.E.1 with +active links or immediate access to the full terms of the Project +Gutenberg-tm License. + +1.E.6. You may convert to and distribute this work in any binary, +compressed, marked up, nonproprietary or proprietary form, including any +word processing or hypertext form. However, if you provide access to or +distribute copies of a Project Gutenberg-tm work in a format other than +"Plain Vanilla ASCII" or other format used in the official version +posted on the official Project Gutenberg-tm web site (www.gutenberg.org), +you must, at no additional cost, fee or expense to the user, provide a +copy, a means of exporting a copy, or a means of obtaining a copy upon +request, of the work in its original "Plain Vanilla ASCII" or other +form. Any alternate format must include the full Project Gutenberg-tm +License as specified in paragraph 1.E.1. + +1.E.7. Do not charge a fee for access to, viewing, displaying, +performing, copying or distributing any Project Gutenberg-tm works +unless you comply with paragraph 1.E.8 or 1.E.9. + +1.E.8. You may charge a reasonable fee for copies of or providing +access to or distributing Project Gutenberg-tm electronic works provided +that + +- You pay a royalty fee of 20% of the gross profits you derive from + the use of Project Gutenberg-tm works calculated using the method + you already use to calculate your applicable taxes. The fee is + owed to the owner of the Project Gutenberg-tm trademark, but he + has agreed to donate royalties under this paragraph to the + Project Gutenberg Literary Archive Foundation. Royalty payments + must be paid within 60 days following each date on which you + prepare (or are legally required to prepare) your periodic tax + returns. Royalty payments should be clearly marked as such and + sent to the Project Gutenberg Literary Archive Foundation at the + address specified in Section 4, "Information about donations to + the Project Gutenberg Literary Archive Foundation." + +- You provide a full refund of any money paid by a user who notifies + you in writing (or by e-mail) within 30 days of receipt that s/he + does not agree to the terms of the full Project Gutenberg-tm + License. You must require such a user to return or + destroy all copies of the works possessed in a physical medium + and discontinue all use of and all access to other copies of + Project Gutenberg-tm works. + +- You provide, in accordance with paragraph 1.F.3, a full refund of any + money paid for a work or a replacement copy, if a defect in the + electronic work is discovered and reported to you within 90 days + of receipt of the work. + +- You comply with all other terms of this agreement for free + distribution of Project Gutenberg-tm works. + +1.E.9. If you wish to charge a fee or distribute a Project Gutenberg-tm +electronic work or group of works on different terms than are set +forth in this agreement, you must obtain permission in writing from +both the Project Gutenberg Literary Archive Foundation and Michael +Hart, the owner of the Project Gutenberg-tm trademark. Contact the +Foundation as set forth in Section 3 below. + +1.F. + +1.F.1. Project Gutenberg volunteers and employees expend considerable +effort to identify, do copyright research on, transcribe and proofread +public domain works in creating the Project Gutenberg-tm +collection. Despite these efforts, Project Gutenberg-tm electronic +works, and the medium on which they may be stored, may contain +"Defects," such as, but not limited to, incomplete, inaccurate or +corrupt data, transcription errors, a copyright or other intellectual +property infringement, a defective or damaged disk or other medium, a +computer virus, or computer codes that damage or cannot be read by +your equipment. + +1.F.2. LIMITED WARRANTY, DISCLAIMER OF DAMAGES - Except for the "Right +of Replacement or Refund" described in paragraph 1.F.3, the Project +Gutenberg Literary Archive Foundation, the owner of the Project +Gutenberg-tm trademark, and any other party distributing a Project +Gutenberg-tm electronic work under this agreement, disclaim all +liability to you for damages, costs and expenses, including legal +fees. YOU AGREE THAT YOU HAVE NO REMEDIES FOR NEGLIGENCE, STRICT +LIABILITY, BREACH OF WARRANTY OR BREACH OF CONTRACT EXCEPT THOSE +PROVIDED IN PARAGRAPH 1.F.3. YOU AGREE THAT THE FOUNDATION, THE +TRADEMARK OWNER, AND ANY DISTRIBUTOR UNDER THIS AGREEMENT WILL NOT BE +LIABLE TO YOU FOR ACTUAL, DIRECT, INDIRECT, CONSEQUENTIAL, PUNITIVE OR +INCIDENTAL DAMAGES EVEN IF YOU GIVE NOTICE OF THE POSSIBILITY OF SUCH +DAMAGE. + +1.F.3. LIMITED RIGHT OF REPLACEMENT OR REFUND - If you discover a +defect in this electronic work within 90 days of receiving it, you can +receive a refund of the money (if any) you paid for it by sending a +written explanation to the person you received the work from. If you +received the work on a physical medium, you must return the medium with +your written explanation. The person or entity that provided you with +the defective work may elect to provide a replacement copy in lieu of a +refund. If you received the work electronically, the person or entity +providing it to you may choose to give you a second opportunity to +receive the work electronically in lieu of a refund. If the second copy +is also defective, you may demand a refund in writing without further +opportunities to fix the problem. + +1.F.4. Except for the limited right of replacement or refund set forth +in paragraph 1.F.3, this work is provided to you 'AS-IS,' WITH NO OTHER +WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +WARRANTIES OF MERCHANTIBILITY OR FITNESS FOR ANY PURPOSE. + +1.F.5. Some states do not allow disclaimers of certain implied +warranties or the exclusion or limitation of certain types of damages. +If any disclaimer or limitation set forth in this agreement violates the +law of the state applicable to this agreement, the agreement shall be +interpreted to make the maximum disclaimer or limitation permitted by +the applicable state law. The invalidity or unenforceability of any +provision of this agreement shall not void the remaining provisions. + +1.F.6. INDEMNITY - You agree to indemnify and hold the Foundation, the +trademark owner, any agent or employee of the Foundation, anyone +providing copies of Project Gutenberg-tm electronic works in accordance +with this agreement, and any volunteers associated with the production, +promotion and distribution of Project Gutenberg-tm electronic works, +harmless from all liability, costs and expenses, including legal fees, +that arise directly or indirectly from any of the following which you do +or cause to occur: (a) distribution of this or any Project Gutenberg-tm +work, (b) alteration, modification, or additions or deletions to any +Project Gutenberg-tm work, and (c) any Defect you cause. + + +Section 2. Information about the Mission of Project Gutenberg-tm + +Project Gutenberg-tm is synonymous with the free distribution of +electronic works in formats readable by the widest variety of computers +including obsolete, old, middle-aged and new computers. It exists +because of the efforts of hundreds of volunteers and donations from +people in all walks of life. + +Volunteers and financial support to provide volunteers with the +assistance they need are critical to reaching Project Gutenberg-tm's +goals and ensuring that the Project Gutenberg-tm collection will +remain freely available for generations to come. In 2001, the Project +Gutenberg Literary Archive Foundation was created to provide a secure +and permanent future for Project Gutenberg-tm and future generations. +To learn more about the Project Gutenberg Literary Archive Foundation +and how your efforts and donations can help, see Sections 3 and 4 +and the Foundation web page at http://www.pglaf.org. + + +Section 3. Information about the Project Gutenberg Literary Archive +Foundation + +The Project Gutenberg Literary Archive Foundation is a non profit +501(c)(3) educational corporation organized under the laws of the +state of Mississippi and granted tax exempt status by the Internal +Revenue Service. The Foundation's EIN or federal tax identification +number is 64-6221541. Its 501(c)(3) letter is posted at +http://pglaf.org/fundraising. Contributions to the Project Gutenberg +Literary Archive Foundation are tax deductible to the full extent +permitted by U.S. federal laws and your state's laws. + +The Foundation's principal office is located at 4557 Melan Dr. S. +Fairbanks, AK, 99712., but its volunteers and employees are scattered +throughout numerous locations. Its business office is located at +809 North 1500 West, Salt Lake City, UT 84116, (801) 596-1887, email +business@pglaf.org. Email contact links and up to date contact +information can be found at the Foundation's web site and official +page at http://pglaf.org + +For additional contact information: + Dr. Gregory B. Newby + Chief Executive and Director + gbnewby@pglaf.org + +Section 4. Information about Donations to the Project Gutenberg +Literary Archive Foundation + +Project Gutenberg-tm depends upon and cannot survive without wide +spread public support and donations to carry out its mission of +increasing the number of public domain and licensed works that can be +freely distributed in machine readable form accessible by the widest +array of equipment including outdated equipment. Many small donations +($1 to $5,000) are particularly important to maintaining tax exempt +status with the IRS. + +The Foundation is committed to complying with the laws regulating +charities and charitable donations in all 50 states of the United +States. Compliance requirements are not uniform and it takes a +considerable effort, much paperwork and many fees to meet and keep up +with these requirements. We do not solicit donations in locations +where we have not received written confirmation of compliance. To +SEND DONATIONS or determine the status of compliance for any +particular state visit http://pglaf.org + +While we cannot and do not solicit contributions from states where we +have not met the solicitation requirements, we know of no prohibition +against accepting unsolicited donations from donors in such states who +approach us with offers to donate. + +International donations are gratefully accepted, but we cannot make +any statements concerning tax treatment of donations received from +outside the United States. U.S. laws alone swamp our small staff. + +Please check the Project Gutenberg Web pages for current donation +methods and addresses. Donations are accepted in a number of other +ways including checks, online payments and credit card donations. +To donate, please visit: http://pglaf.org/donate + + +Section 5. General Information About Project Gutenberg-tm electronic +works. + +Professor Michael S. Hart is the originator of the Project Gutenberg-tm +concept of a library of electronic works that could be freely shared +with anyone. For thirty years, he produced and distributed Project +Gutenberg-tm eBooks with only a loose network of volunteer support. + +Project Gutenberg-tm eBooks are often created from several printed +editions, all of which are confirmed as Public Domain in the U.S. +unless a copyright notice is included. Thus, we do not necessarily +keep eBooks in compliance with any particular paper edition. + +Each eBook is in a subdirectory of the same number as the eBook's +eBook number, often in several formats including plain vanilla ASCII, +compressed (zipped), HTML and others. + +Corrected EDITIONS of our eBooks replace the old file and take over +the old filename and etext number. The replaced older file is renamed. +VERSIONS based on separate sources are treated as new eBooks receiving +new filenames and etext numbers. + +Most people start at our Web site which has the main PG search facility: + +http://www.gutenberg.org + +This Web site includes information about Project Gutenberg-tm, +including how to make donations to the Project Gutenberg Literary +Archive Foundation, how to help produce our new eBooks, and how to +subscribe to our email newsletter to hear about new eBooks. + +EBooks posted prior to November 2003, with eBook numbers BELOW #10000, +are filed in directories based on their release date. If you want to +download any of these eBooks directly, rather than using the regular +search system you may utilize the following addresses and just +download by the etext year. + +http://www.ibiblio.org/gutenberg/etext06 + + (Or /etext 05, 04, 03, 02, 01, 00, 99, + 98, 97, 96, 95, 94, 93, 92, 92, 91 or 90) + +EBooks posted since November 2003, with etext numbers OVER #10000, are +filed in a different way. The year of a release date is no longer part +of the directory path. The path is based on the etext number (which is +identical to the filename). The path to the file is made up of single +digits corresponding to all but the last digit in the filename. For +example an eBook of filename 10234 would be found at: + +http://www.gutenberg.org/1/0/2/3/10234 + +or filename 24689 would be found at: +http://www.gutenberg.org/2/4/6/8/24689 + +An alternative method of locating eBooks: +http://www.gutenberg.org/GUTINDEX.ALL + +*** END: FULL LICENSE *** diff --git a/testing/tests/sqlcipher/test_backend.py b/testing/tests/sqlcipher/test_backend.py new file mode 100644 index 00000000..11472d46 --- /dev/null +++ b/testing/tests/sqlcipher/test_backend.py @@ -0,0 +1,705 @@ +# -*- coding: utf-8 -*- +# test_sqlcipher.py +# Copyright (C) 2013 LEAP +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +""" +Test sqlcipher backend internals. +""" +import os +import time +import threading +import tempfile +import shutil + +from pysqlcipher import dbapi2 +from testscenarios import TestWithScenarios + +# l2db stuff. +from leap.soledad.common.l2db import errors +from leap.soledad.common.l2db import query_parser +from leap.soledad.common.l2db.backends.sqlite_backend \ + import SQLitePartialExpandDatabase + +# soledad stuff. +from leap.soledad.common import soledad_assert +from leap.soledad.common.document import SoledadDocument +from leap.soledad.client.sqlcipher import SQLCipherDatabase +from leap.soledad.client.sqlcipher import SQLCipherOptions +from leap.soledad.client.sqlcipher import DatabaseIsNotEncrypted + +# u1db tests stuff. +from test_soledad import u1db_tests as tests +from test_soledad.u1db_tests import test_backends +from test_soledad.u1db_tests import test_open +from test_soledad.util import SQLCIPHER_SCENARIOS +from test_soledad.util import PASSWORD +from test_soledad.util import BaseSoledadTest + + +def sqlcipher_open(path, passphrase, create=True, document_factory=None): + return SQLCipherDatabase( + SQLCipherOptions(path, passphrase, create=create)) + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_common_backend`. +# ----------------------------------------------------------------------------- + +class TestSQLCipherBackendImpl(tests.TestCase): + + def test__allocate_doc_id(self): + db = sqlcipher_open(':memory:', PASSWORD) + doc_id1 = db._allocate_doc_id() + self.assertTrue(doc_id1.startswith('D-')) + self.assertEqual(34, len(doc_id1)) + int(doc_id1[len('D-'):], 16) + self.assertNotEqual(doc_id1, db._allocate_doc_id()) + db.close() + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_backends`. +# ----------------------------------------------------------------------------- + +class SQLCipherTests(TestWithScenarios, test_backends.AllDatabaseTests): + scenarios = SQLCIPHER_SCENARIOS + + +class SQLCipherDatabaseTests(TestWithScenarios, + test_backends.LocalDatabaseTests): + scenarios = SQLCIPHER_SCENARIOS + + +class SQLCipherValidateGenNTransIdTests( + TestWithScenarios, + test_backends.LocalDatabaseValidateGenNTransIdTests): + scenarios = SQLCIPHER_SCENARIOS + + +class SQLCipherValidateSourceGenTests( + TestWithScenarios, + test_backends.LocalDatabaseValidateSourceGenTests): + scenarios = SQLCIPHER_SCENARIOS + + +class SQLCipherWithConflictsTests( + TestWithScenarios, + test_backends.LocalDatabaseWithConflictsTests): + scenarios = SQLCIPHER_SCENARIOS + + +class SQLCipherIndexTests( + TestWithScenarios, test_backends.DatabaseIndexTests): + scenarios = SQLCIPHER_SCENARIOS + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_sqlite_backend`. +# ----------------------------------------------------------------------------- + +class TestSQLCipherDatabase(tests.TestCase): + """ + Tests from u1db.tests.test_sqlite_backend.TestSQLiteDatabase. + """ + + def test_atomic_initialize(self): + # This test was modified to ensure that db2.close() is called within + # the thread that created the database. + tmpdir = self.createTempDir() + dbname = os.path.join(tmpdir, 'atomic.db') + + t2 = None # will be a thread + + class SQLCipherDatabaseTesting(SQLCipherDatabase): + _index_storage_value = "testing" + + def __init__(self, dbname, ntry): + self._try = ntry + self._is_initialized_invocations = 0 + SQLCipherDatabase.__init__( + self, + SQLCipherOptions(dbname, PASSWORD)) + + def _is_initialized(self, c): + res = \ + SQLCipherDatabase._is_initialized(self, c) + if self._try == 1: + self._is_initialized_invocations += 1 + if self._is_initialized_invocations == 2: + t2.start() + # hard to do better and have a generic test + time.sleep(0.05) + return res + + class SecondTry(threading.Thread): + + outcome2 = [] + + def run(self): + try: + db2 = SQLCipherDatabaseTesting(dbname, 2) + except Exception, e: + SecondTry.outcome2.append(e) + else: + SecondTry.outcome2.append(db2) + + t2 = SecondTry() + db1 = SQLCipherDatabaseTesting(dbname, 1) + t2.join() + + self.assertIsInstance(SecondTry.outcome2[0], SQLCipherDatabaseTesting) + self.assertTrue(db1._is_initialized(db1._get_sqlite_handle().cursor())) + db1.close() + + +class TestSQLCipherPartialExpandDatabase(tests.TestCase): + """ + Tests from u1db.tests.test_sqlite_backend.TestSQLitePartialExpandDatabase. + """ + + # The following tests had to be cloned from u1db because they all + # instantiate the backend directly, so we need to change that in order to + # our backend be instantiated in place. + + def setUp(self): + self.db = sqlcipher_open(':memory:', PASSWORD) + + def tearDown(self): + self.db.close() + + def test_default_replica_uid(self): + self.assertIsNot(None, self.db._replica_uid) + self.assertEqual(32, len(self.db._replica_uid)) + int(self.db._replica_uid, 16) + + def test__parse_index(self): + g = self.db._parse_index_definition('fieldname') + self.assertIsInstance(g, query_parser.ExtractField) + self.assertEqual(['fieldname'], g.field) + + def test__update_indexes(self): + g = self.db._parse_index_definition('fieldname') + c = self.db._get_sqlite_handle().cursor() + self.db._update_indexes('doc-id', {'fieldname': 'val'}, + [('fieldname', g)], c) + c.execute('SELECT doc_id, field_name, value FROM document_fields') + self.assertEqual([('doc-id', 'fieldname', 'val')], + c.fetchall()) + + def test_create_database(self): + raw_db = self.db._get_sqlite_handle() + self.assertNotEqual(None, raw_db) + + def test__set_replica_uid(self): + # Start from scratch, so that replica_uid isn't set. + self.assertIsNot(None, self.db._real_replica_uid) + self.assertIsNot(None, self.db._replica_uid) + self.db._set_replica_uid('foo') + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT value FROM u1db_config WHERE name='replica_uid'") + self.assertEqual(('foo',), c.fetchone()) + self.assertEqual('foo', self.db._real_replica_uid) + self.assertEqual('foo', self.db._replica_uid) + self.db._close_sqlite_handle() + self.assertEqual('foo', self.db._replica_uid) + + def test__open_database(self): + # SQLCipherDatabase has no _open_database() method, so we just pass + # (and test for the same funcionality on test_open_database_existing() + # below). + pass + + def test__open_database_with_factory(self): + # SQLCipherDatabase has no _open_database() method. + pass + + def test__open_database_non_existent(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/non-existent.sqlite' + self.assertRaises(errors.DatabaseDoesNotExist, + sqlcipher_open, + path, PASSWORD, create=False) + + def test__open_database_during_init(self): + # The purpose of this test is to ensure that _open_database() parallel + # db initialization behaviour is correct. As SQLCipherDatabase does + # not have an _open_database() method, we just do not implement this + # test. + pass + + def test__open_database_invalid(self): + # This test was modified to ensure that an empty database file will + # raise a DatabaseIsNotEncrypted exception instead of a + # dbapi2.OperationalError exception. + temp_dir = self.createTempDir(prefix='u1db-test-') + path1 = temp_dir + '/invalid1.db' + with open(path1, 'wb') as f: + f.write("") + self.assertRaises(DatabaseIsNotEncrypted, + sqlcipher_open, path1, + PASSWORD) + with open(path1, 'wb') as f: + f.write("invalid") + self.assertRaises(dbapi2.DatabaseError, + sqlcipher_open, path1, + PASSWORD) + + def test_open_database_existing(self): + # In the context of SQLCipherDatabase, where no _open_database() + # method exists and thus there's no call to _which_index_storage(), + # this test tests for the same functionality as + # test_open_database_create() below. So, we just pass. + pass + + def test_open_database_with_factory(self): + # SQLCipherDatabase's constructor has no factory parameter. + pass + + def test_open_database_create(self): + # SQLCipherDatabas has no open_database() method, so we just test for + # the actual database constructor effects. + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/new.sqlite' + db1 = sqlcipher_open(path, PASSWORD, create=True) + db2 = sqlcipher_open(path, PASSWORD, create=False) + self.assertIsInstance(db2, SQLCipherDatabase) + db1.close() + db2.close() + + def test_create_database_initializes_schema(self): + # This test had to be cloned because our implementation of SQLCipher + # backend is referenced with an index_storage_value that includes the + # word "encrypted". See u1db's sqlite_backend and our + # sqlcipher_backend for reference. + raw_db = self.db._get_sqlite_handle() + c = raw_db.cursor() + c.execute("SELECT * FROM u1db_config") + config = dict([(r[0], r[1]) for r in c.fetchall()]) + replica_uid = self.db._replica_uid + self.assertEqual({'sql_schema': '0', 'replica_uid': replica_uid, + 'index_storage': 'expand referenced encrypted'}, + config) + + def test_store_syncable(self): + doc = self.db.create_doc_from_json(tests.simple_doc) + # assert that docs are syncable by default + self.assertEqual(True, doc.syncable) + # assert that we can store syncable = False + doc.syncable = False + self.db.put_doc(doc) + self.assertEqual(False, self.db.get_doc(doc.doc_id).syncable) + # assert that we can store syncable = True + doc.syncable = True + self.db.put_doc(doc) + self.assertEqual(True, self.db.get_doc(doc.doc_id).syncable) + + def test__close_sqlite_handle(self): + raw_db = self.db._get_sqlite_handle() + self.db._close_sqlite_handle() + self.assertRaises(dbapi2.ProgrammingError, + raw_db.cursor) + + def test__get_generation(self): + self.assertEqual(0, self.db._get_generation()) + + def test__get_generation_info(self): + self.assertEqual((0, ''), self.db._get_generation_info()) + + def test_create_index(self): + self.db.create_index('test-idx', "key") + self.assertEqual([('test-idx', ["key"])], self.db.list_indexes()) + + def test_create_index_multiple_fields(self): + self.db.create_index('test-idx', "key", "key2") + self.assertEqual([('test-idx', ["key", "key2"])], + self.db.list_indexes()) + + def test__get_index_definition(self): + self.db.create_index('test-idx', "key", "key2") + # TODO: How would you test that an index is getting used for an SQL + # request? + self.assertEqual(["key", "key2"], + self.db._get_index_definition('test-idx')) + + def test_list_index_mixed(self): + # Make sure that we properly order the output + c = self.db._get_sqlite_handle().cursor() + # We intentionally insert the data in weird ordering, to make sure the + # query still gets it back correctly. + c.executemany("INSERT INTO index_definitions VALUES (?, ?, ?)", + [('idx-1', 0, 'key10'), + ('idx-2', 2, 'key22'), + ('idx-1', 1, 'key11'), + ('idx-2', 0, 'key20'), + ('idx-2', 1, 'key21')]) + self.assertEqual([('idx-1', ['key10', 'key11']), + ('idx-2', ['key20', 'key21', 'key22'])], + self.db.list_indexes()) + + def test_no_indexes_no_document_fields(self): + self.db.create_doc_from_json( + '{"key1": "val1", "key2": "val2"}') + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([], c.fetchall()) + + def test_create_extracts_fields(self): + doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') + doc2 = self.db.create_doc_from_json('{"key1": "valx", "key2": "valy"}') + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([], c.fetchall()) + self.db.create_index('test', 'key1', 'key2') + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual(sorted( + [(doc1.doc_id, "key1", "val1"), + (doc1.doc_id, "key2", "val2"), + (doc2.doc_id, "key1", "valx"), + (doc2.doc_id, "key2", "valy"), ]), sorted(c.fetchall())) + + def test_put_updates_fields(self): + self.db.create_index('test', 'key1', 'key2') + doc1 = self.db.create_doc_from_json( + '{"key1": "val1", "key2": "val2"}') + doc1.content = {"key1": "val1", "key2": "valy"} + self.db.put_doc(doc1) + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([(doc1.doc_id, "key1", "val1"), + (doc1.doc_id, "key2", "valy"), ], c.fetchall()) + + def test_put_updates_nested_fields(self): + self.db.create_index('test', 'key', 'sub.doc') + doc1 = self.db.create_doc_from_json(tests.nested_doc) + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([(doc1.doc_id, "key", "value"), + (doc1.doc_id, "sub.doc", "underneath"), ], + c.fetchall()) + + def test__ensure_schema_rollback(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/rollback.db' + + class SQLitePartialExpandDbTesting(SQLCipherDatabase): + + def _set_replica_uid_in_transaction(self, uid): + super(SQLitePartialExpandDbTesting, + self)._set_replica_uid_in_transaction(uid) + if fail: + raise Exception() + + db = SQLitePartialExpandDbTesting.__new__(SQLitePartialExpandDbTesting) + db._db_handle = dbapi2.connect(path) # db is there but not yet init-ed + fail = True + self.assertRaises(Exception, db._ensure_schema) + fail = False + db._initialize(db._db_handle.cursor()) + + def test_open_database_non_existent(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/non-existent.sqlite' + self.assertRaises(errors.DatabaseDoesNotExist, + sqlcipher_open, path, "123", + create=False) + + def test_delete_database_existent(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/new.sqlite' + db = sqlcipher_open(path, "123", create=True) + db.close() + SQLCipherDatabase.delete_database(path) + self.assertRaises(errors.DatabaseDoesNotExist, + sqlcipher_open, path, "123", + create=False) + + def test_delete_database_nonexistent(self): + temp_dir = self.createTempDir(prefix='u1db-test-') + path = temp_dir + '/non-existent.sqlite' + self.assertRaises(errors.DatabaseDoesNotExist, + SQLCipherDatabase.delete_database, path) + + def test__get_indexed_fields(self): + self.db.create_index('idx1', 'a', 'b') + self.assertEqual(set(['a', 'b']), self.db._get_indexed_fields()) + self.db.create_index('idx2', 'b', 'c') + self.assertEqual(set(['a', 'b', 'c']), self.db._get_indexed_fields()) + + def test_indexed_fields_expanded(self): + self.db.create_index('idx1', 'key1') + doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') + self.assertEqual(set(['key1']), self.db._get_indexed_fields()) + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([(doc1.doc_id, 'key1', 'val1')], c.fetchall()) + + def test_create_index_updates_fields(self): + doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') + self.db.create_index('idx1', 'key1') + self.assertEqual(set(['key1']), self.db._get_indexed_fields()) + c = self.db._get_sqlite_handle().cursor() + c.execute("SELECT doc_id, field_name, value FROM document_fields" + " ORDER BY doc_id, field_name, value") + self.assertEqual([(doc1.doc_id, 'key1', 'val1')], c.fetchall()) + + def assertFormatQueryEquals(self, exp_statement, exp_args, definition, + values): + statement, args = self.db._format_query(definition, values) + self.assertEqual(exp_statement, statement) + self.assertEqual(exp_args, args) + + def test__format_query(self): + self.assertFormatQueryEquals( + "SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM " + "document d, document_fields d0 LEFT OUTER JOIN conflicts c ON " + "c.doc_id = d.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name " + "= ? AND d0.value = ? GROUP BY d.doc_id, d.doc_rev, d.content " + "ORDER BY d0.value;", ["key1", "a"], + ["key1"], ["a"]) + + def test__format_query2(self): + self.assertFormatQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value = ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value = ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value = ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' + 'd0.value, d1.value, d2.value;', + ["key1", "a", "key2", "b", "key3", "c"], + ["key1", "key2", "key3"], ["a", "b", "c"]) + + def test__format_query_wildcard(self): + self.assertFormatQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value = ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value GLOB ? AND d.doc_id = d2.doc_id AND d2.field_name = ? ' + 'AND d2.value NOT NULL GROUP BY d.doc_id, d.doc_rev, d.content ' + 'ORDER BY d0.value, d1.value, d2.value;', + ["key1", "a", "key2", "b*", "key3"], ["key1", "key2", "key3"], + ["a", "b*", "*"]) + + def assertFormatRangeQueryEquals(self, exp_statement, exp_args, definition, + start_value, end_value): + statement, args = self.db._format_range_query( + definition, start_value, end_value) + self.assertEqual(exp_statement, statement) + self.assertEqual(exp_args, args) + + def test__format_range_query(self): + self.assertFormatRangeQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value >= ? AND d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value <= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value <= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' + 'd0.value, d1.value, d2.value;', + ['key1', 'a', 'key2', 'b', 'key3', 'c', 'key1', 'p', 'key2', 'q', + 'key3', 'r'], + ["key1", "key2", "key3"], ["a", "b", "c"], ["p", "q", "r"]) + + def test__format_range_query_no_start(self): + self.assertFormatRangeQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value <= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value <= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' + 'd0.value, d1.value, d2.value;', + ['key1', 'a', 'key2', 'b', 'key3', 'c'], + ["key1", "key2", "key3"], None, ["a", "b", "c"]) + + def test__format_range_query_no_end(self): + self.assertFormatRangeQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value >= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' + 'd0.value, d1.value, d2.value;', + ['key1', 'a', 'key2', 'b', 'key3', 'c'], + ["key1", "key2", "key3"], ["a", "b", "c"], None) + + def test__format_range_query_wildcard(self): + self.assertFormatRangeQueryEquals( + 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' + 'document d, document_fields d0, document_fields d1, ' + 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' + 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' + 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' + 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' + 'd2.value NOT NULL AND d.doc_id = d0.doc_id AND d0.field_name = ? ' + 'AND d0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? ' + 'AND (d1.value < ? OR d1.value GLOB ?) AND d.doc_id = d2.doc_id ' + 'AND d2.field_name = ? AND d2.value NOT NULL GROUP BY d.doc_id, ' + 'd.doc_rev, d.content ORDER BY d0.value, d1.value, d2.value;', + ['key1', 'a', 'key2', 'b', 'key3', 'key1', 'p', 'key2', 'q', 'q*', + 'key3'], + ["key1", "key2", "key3"], ["a", "b*", "*"], ["p", "q*", "*"]) + + +# ----------------------------------------------------------------------------- +# The following tests come from `u1db.tests.test_open`. +# ----------------------------------------------------------------------------- + + +class SQLCipherOpen(test_open.TestU1DBOpen): + + def test_open_no_create(self): + self.assertRaises(errors.DatabaseDoesNotExist, + sqlcipher_open, self.db_path, + PASSWORD, + create=False) + self.assertFalse(os.path.exists(self.db_path)) + + def test_open_create(self): + db = sqlcipher_open(self.db_path, PASSWORD, create=True) + self.addCleanup(db.close) + self.assertTrue(os.path.exists(self.db_path)) + self.assertIsInstance(db, SQLCipherDatabase) + + def test_open_with_factory(self): + db = sqlcipher_open(self.db_path, PASSWORD, create=True, + document_factory=SoledadDocument) + self.addCleanup(db.close) + doc = db.create_doc({}) + self.assertTrue(isinstance(doc, SoledadDocument)) + + def test_open_existing(self): + db = sqlcipher_open(self.db_path, PASSWORD) + self.addCleanup(db.close) + doc = db.create_doc_from_json(tests.simple_doc) + # Even though create=True, we shouldn't wipe the db + db2 = sqlcipher_open(self.db_path, PASSWORD, create=True) + self.addCleanup(db2.close) + doc2 = db2.get_doc(doc.doc_id) + self.assertEqual(doc, doc2) + + def test_open_existing_no_create(self): + db = sqlcipher_open(self.db_path, PASSWORD) + self.addCleanup(db.close) + db2 = sqlcipher_open(self.db_path, PASSWORD, create=False) + self.addCleanup(db2.close) + self.assertIsInstance(db2, SQLCipherDatabase) + + +# ----------------------------------------------------------------------------- +# Tests for actual encryption of the database +# ----------------------------------------------------------------------------- + +class SQLCipherEncryptionTests(BaseSoledadTest): + + """ + Tests to guarantee SQLCipher is indeed encrypting data when storing. + """ + + def _delete_dbfiles(self): + for dbfile in [self.DB_FILE]: + if os.path.exists(dbfile): + os.unlink(dbfile) + + def setUp(self): + # the following come from BaseLeapTest.setUpClass, because + # twisted.trial doesn't support such class methods for setting up + # test classes. + self.old_path = os.environ['PATH'] + self.old_home = os.environ['HOME'] + self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") + self.home = self.tempdir + bin_tdir = os.path.join( + self.tempdir, + 'bin') + os.environ["PATH"] = bin_tdir + os.environ["HOME"] = self.tempdir + # this is our own stuff + self.DB_FILE = os.path.join(self.tempdir, 'test.db') + self._delete_dbfiles() + + def tearDown(self): + self._delete_dbfiles() + # the following come from BaseLeapTest.tearDownClass, because + # twisted.trial doesn't support such class methods for tearing down + # test classes. + os.environ["PATH"] = self.old_path + os.environ["HOME"] = self.old_home + # safety check! please do not wipe my home... + # XXX needs to adapt to non-linuces + soledad_assert( + self.tempdir.startswith('/tmp/leap_tests-') or + self.tempdir.startswith('/var/folder'), + "beware! tried to remove a dir which does not " + "live in temporal folder!") + shutil.rmtree(self.tempdir) + + def test_try_to_open_encrypted_db_with_sqlite_backend(self): + """ + SQLite backend should not succeed to open SQLCipher databases. + """ + db = sqlcipher_open(self.DB_FILE, PASSWORD) + doc = db.create_doc_from_json(tests.simple_doc) + db.close() + try: + # trying to open an encrypted database with the regular u1db + # backend should raise a DatabaseError exception. + SQLitePartialExpandDatabase(self.DB_FILE, + document_factory=SoledadDocument) + raise DatabaseIsNotEncrypted() + except dbapi2.DatabaseError: + # at this point we know that the regular U1DB sqlcipher backend + # did not succeed on opening the database, so it was indeed + # encrypted. + db = sqlcipher_open(self.DB_FILE, PASSWORD) + doc = db.get_doc(doc.doc_id) + self.assertEqual(tests.simple_doc, doc.get_json(), + 'decrypted content mismatch') + db.close() + + def test_try_to_open_raw_db_with_sqlcipher_backend(self): + """ + SQLCipher backend should not succeed to open unencrypted databases. + """ + db = SQLitePartialExpandDatabase(self.DB_FILE, + document_factory=SoledadDocument) + db.create_doc_from_json(tests.simple_doc) + db.close() + try: + # trying to open the a non-encrypted database with sqlcipher + # backend should raise a DatabaseIsNotEncrypted exception. + db = sqlcipher_open(self.DB_FILE, PASSWORD) + db.close() + raise dbapi2.DatabaseError( + "SQLCipher backend should not be able to open non-encrypted " + "dbs.") + except DatabaseIsNotEncrypted: + pass diff --git a/testing/tests/sqlcipher/test_sqlcipher.py b/testing/tests/sqlcipher/test_sqlcipher.py deleted file mode 100644 index 11472d46..00000000 --- a/testing/tests/sqlcipher/test_sqlcipher.py +++ /dev/null @@ -1,705 +0,0 @@ -# -*- coding: utf-8 -*- -# test_sqlcipher.py -# Copyright (C) 2013 LEAP -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -""" -Test sqlcipher backend internals. -""" -import os -import time -import threading -import tempfile -import shutil - -from pysqlcipher import dbapi2 -from testscenarios import TestWithScenarios - -# l2db stuff. -from leap.soledad.common.l2db import errors -from leap.soledad.common.l2db import query_parser -from leap.soledad.common.l2db.backends.sqlite_backend \ - import SQLitePartialExpandDatabase - -# soledad stuff. -from leap.soledad.common import soledad_assert -from leap.soledad.common.document import SoledadDocument -from leap.soledad.client.sqlcipher import SQLCipherDatabase -from leap.soledad.client.sqlcipher import SQLCipherOptions -from leap.soledad.client.sqlcipher import DatabaseIsNotEncrypted - -# u1db tests stuff. -from test_soledad import u1db_tests as tests -from test_soledad.u1db_tests import test_backends -from test_soledad.u1db_tests import test_open -from test_soledad.util import SQLCIPHER_SCENARIOS -from test_soledad.util import PASSWORD -from test_soledad.util import BaseSoledadTest - - -def sqlcipher_open(path, passphrase, create=True, document_factory=None): - return SQLCipherDatabase( - SQLCipherOptions(path, passphrase, create=create)) - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_common_backend`. -# ----------------------------------------------------------------------------- - -class TestSQLCipherBackendImpl(tests.TestCase): - - def test__allocate_doc_id(self): - db = sqlcipher_open(':memory:', PASSWORD) - doc_id1 = db._allocate_doc_id() - self.assertTrue(doc_id1.startswith('D-')) - self.assertEqual(34, len(doc_id1)) - int(doc_id1[len('D-'):], 16) - self.assertNotEqual(doc_id1, db._allocate_doc_id()) - db.close() - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_backends`. -# ----------------------------------------------------------------------------- - -class SQLCipherTests(TestWithScenarios, test_backends.AllDatabaseTests): - scenarios = SQLCIPHER_SCENARIOS - - -class SQLCipherDatabaseTests(TestWithScenarios, - test_backends.LocalDatabaseTests): - scenarios = SQLCIPHER_SCENARIOS - - -class SQLCipherValidateGenNTransIdTests( - TestWithScenarios, - test_backends.LocalDatabaseValidateGenNTransIdTests): - scenarios = SQLCIPHER_SCENARIOS - - -class SQLCipherValidateSourceGenTests( - TestWithScenarios, - test_backends.LocalDatabaseValidateSourceGenTests): - scenarios = SQLCIPHER_SCENARIOS - - -class SQLCipherWithConflictsTests( - TestWithScenarios, - test_backends.LocalDatabaseWithConflictsTests): - scenarios = SQLCIPHER_SCENARIOS - - -class SQLCipherIndexTests( - TestWithScenarios, test_backends.DatabaseIndexTests): - scenarios = SQLCIPHER_SCENARIOS - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_sqlite_backend`. -# ----------------------------------------------------------------------------- - -class TestSQLCipherDatabase(tests.TestCase): - """ - Tests from u1db.tests.test_sqlite_backend.TestSQLiteDatabase. - """ - - def test_atomic_initialize(self): - # This test was modified to ensure that db2.close() is called within - # the thread that created the database. - tmpdir = self.createTempDir() - dbname = os.path.join(tmpdir, 'atomic.db') - - t2 = None # will be a thread - - class SQLCipherDatabaseTesting(SQLCipherDatabase): - _index_storage_value = "testing" - - def __init__(self, dbname, ntry): - self._try = ntry - self._is_initialized_invocations = 0 - SQLCipherDatabase.__init__( - self, - SQLCipherOptions(dbname, PASSWORD)) - - def _is_initialized(self, c): - res = \ - SQLCipherDatabase._is_initialized(self, c) - if self._try == 1: - self._is_initialized_invocations += 1 - if self._is_initialized_invocations == 2: - t2.start() - # hard to do better and have a generic test - time.sleep(0.05) - return res - - class SecondTry(threading.Thread): - - outcome2 = [] - - def run(self): - try: - db2 = SQLCipherDatabaseTesting(dbname, 2) - except Exception, e: - SecondTry.outcome2.append(e) - else: - SecondTry.outcome2.append(db2) - - t2 = SecondTry() - db1 = SQLCipherDatabaseTesting(dbname, 1) - t2.join() - - self.assertIsInstance(SecondTry.outcome2[0], SQLCipherDatabaseTesting) - self.assertTrue(db1._is_initialized(db1._get_sqlite_handle().cursor())) - db1.close() - - -class TestSQLCipherPartialExpandDatabase(tests.TestCase): - """ - Tests from u1db.tests.test_sqlite_backend.TestSQLitePartialExpandDatabase. - """ - - # The following tests had to be cloned from u1db because they all - # instantiate the backend directly, so we need to change that in order to - # our backend be instantiated in place. - - def setUp(self): - self.db = sqlcipher_open(':memory:', PASSWORD) - - def tearDown(self): - self.db.close() - - def test_default_replica_uid(self): - self.assertIsNot(None, self.db._replica_uid) - self.assertEqual(32, len(self.db._replica_uid)) - int(self.db._replica_uid, 16) - - def test__parse_index(self): - g = self.db._parse_index_definition('fieldname') - self.assertIsInstance(g, query_parser.ExtractField) - self.assertEqual(['fieldname'], g.field) - - def test__update_indexes(self): - g = self.db._parse_index_definition('fieldname') - c = self.db._get_sqlite_handle().cursor() - self.db._update_indexes('doc-id', {'fieldname': 'val'}, - [('fieldname', g)], c) - c.execute('SELECT doc_id, field_name, value FROM document_fields') - self.assertEqual([('doc-id', 'fieldname', 'val')], - c.fetchall()) - - def test_create_database(self): - raw_db = self.db._get_sqlite_handle() - self.assertNotEqual(None, raw_db) - - def test__set_replica_uid(self): - # Start from scratch, so that replica_uid isn't set. - self.assertIsNot(None, self.db._real_replica_uid) - self.assertIsNot(None, self.db._replica_uid) - self.db._set_replica_uid('foo') - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT value FROM u1db_config WHERE name='replica_uid'") - self.assertEqual(('foo',), c.fetchone()) - self.assertEqual('foo', self.db._real_replica_uid) - self.assertEqual('foo', self.db._replica_uid) - self.db._close_sqlite_handle() - self.assertEqual('foo', self.db._replica_uid) - - def test__open_database(self): - # SQLCipherDatabase has no _open_database() method, so we just pass - # (and test for the same funcionality on test_open_database_existing() - # below). - pass - - def test__open_database_with_factory(self): - # SQLCipherDatabase has no _open_database() method. - pass - - def test__open_database_non_existent(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/non-existent.sqlite' - self.assertRaises(errors.DatabaseDoesNotExist, - sqlcipher_open, - path, PASSWORD, create=False) - - def test__open_database_during_init(self): - # The purpose of this test is to ensure that _open_database() parallel - # db initialization behaviour is correct. As SQLCipherDatabase does - # not have an _open_database() method, we just do not implement this - # test. - pass - - def test__open_database_invalid(self): - # This test was modified to ensure that an empty database file will - # raise a DatabaseIsNotEncrypted exception instead of a - # dbapi2.OperationalError exception. - temp_dir = self.createTempDir(prefix='u1db-test-') - path1 = temp_dir + '/invalid1.db' - with open(path1, 'wb') as f: - f.write("") - self.assertRaises(DatabaseIsNotEncrypted, - sqlcipher_open, path1, - PASSWORD) - with open(path1, 'wb') as f: - f.write("invalid") - self.assertRaises(dbapi2.DatabaseError, - sqlcipher_open, path1, - PASSWORD) - - def test_open_database_existing(self): - # In the context of SQLCipherDatabase, where no _open_database() - # method exists and thus there's no call to _which_index_storage(), - # this test tests for the same functionality as - # test_open_database_create() below. So, we just pass. - pass - - def test_open_database_with_factory(self): - # SQLCipherDatabase's constructor has no factory parameter. - pass - - def test_open_database_create(self): - # SQLCipherDatabas has no open_database() method, so we just test for - # the actual database constructor effects. - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/new.sqlite' - db1 = sqlcipher_open(path, PASSWORD, create=True) - db2 = sqlcipher_open(path, PASSWORD, create=False) - self.assertIsInstance(db2, SQLCipherDatabase) - db1.close() - db2.close() - - def test_create_database_initializes_schema(self): - # This test had to be cloned because our implementation of SQLCipher - # backend is referenced with an index_storage_value that includes the - # word "encrypted". See u1db's sqlite_backend and our - # sqlcipher_backend for reference. - raw_db = self.db._get_sqlite_handle() - c = raw_db.cursor() - c.execute("SELECT * FROM u1db_config") - config = dict([(r[0], r[1]) for r in c.fetchall()]) - replica_uid = self.db._replica_uid - self.assertEqual({'sql_schema': '0', 'replica_uid': replica_uid, - 'index_storage': 'expand referenced encrypted'}, - config) - - def test_store_syncable(self): - doc = self.db.create_doc_from_json(tests.simple_doc) - # assert that docs are syncable by default - self.assertEqual(True, doc.syncable) - # assert that we can store syncable = False - doc.syncable = False - self.db.put_doc(doc) - self.assertEqual(False, self.db.get_doc(doc.doc_id).syncable) - # assert that we can store syncable = True - doc.syncable = True - self.db.put_doc(doc) - self.assertEqual(True, self.db.get_doc(doc.doc_id).syncable) - - def test__close_sqlite_handle(self): - raw_db = self.db._get_sqlite_handle() - self.db._close_sqlite_handle() - self.assertRaises(dbapi2.ProgrammingError, - raw_db.cursor) - - def test__get_generation(self): - self.assertEqual(0, self.db._get_generation()) - - def test__get_generation_info(self): - self.assertEqual((0, ''), self.db._get_generation_info()) - - def test_create_index(self): - self.db.create_index('test-idx', "key") - self.assertEqual([('test-idx', ["key"])], self.db.list_indexes()) - - def test_create_index_multiple_fields(self): - self.db.create_index('test-idx', "key", "key2") - self.assertEqual([('test-idx', ["key", "key2"])], - self.db.list_indexes()) - - def test__get_index_definition(self): - self.db.create_index('test-idx', "key", "key2") - # TODO: How would you test that an index is getting used for an SQL - # request? - self.assertEqual(["key", "key2"], - self.db._get_index_definition('test-idx')) - - def test_list_index_mixed(self): - # Make sure that we properly order the output - c = self.db._get_sqlite_handle().cursor() - # We intentionally insert the data in weird ordering, to make sure the - # query still gets it back correctly. - c.executemany("INSERT INTO index_definitions VALUES (?, ?, ?)", - [('idx-1', 0, 'key10'), - ('idx-2', 2, 'key22'), - ('idx-1', 1, 'key11'), - ('idx-2', 0, 'key20'), - ('idx-2', 1, 'key21')]) - self.assertEqual([('idx-1', ['key10', 'key11']), - ('idx-2', ['key20', 'key21', 'key22'])], - self.db.list_indexes()) - - def test_no_indexes_no_document_fields(self): - self.db.create_doc_from_json( - '{"key1": "val1", "key2": "val2"}') - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([], c.fetchall()) - - def test_create_extracts_fields(self): - doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') - doc2 = self.db.create_doc_from_json('{"key1": "valx", "key2": "valy"}') - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([], c.fetchall()) - self.db.create_index('test', 'key1', 'key2') - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual(sorted( - [(doc1.doc_id, "key1", "val1"), - (doc1.doc_id, "key2", "val2"), - (doc2.doc_id, "key1", "valx"), - (doc2.doc_id, "key2", "valy"), ]), sorted(c.fetchall())) - - def test_put_updates_fields(self): - self.db.create_index('test', 'key1', 'key2') - doc1 = self.db.create_doc_from_json( - '{"key1": "val1", "key2": "val2"}') - doc1.content = {"key1": "val1", "key2": "valy"} - self.db.put_doc(doc1) - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([(doc1.doc_id, "key1", "val1"), - (doc1.doc_id, "key2", "valy"), ], c.fetchall()) - - def test_put_updates_nested_fields(self): - self.db.create_index('test', 'key', 'sub.doc') - doc1 = self.db.create_doc_from_json(tests.nested_doc) - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([(doc1.doc_id, "key", "value"), - (doc1.doc_id, "sub.doc", "underneath"), ], - c.fetchall()) - - def test__ensure_schema_rollback(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/rollback.db' - - class SQLitePartialExpandDbTesting(SQLCipherDatabase): - - def _set_replica_uid_in_transaction(self, uid): - super(SQLitePartialExpandDbTesting, - self)._set_replica_uid_in_transaction(uid) - if fail: - raise Exception() - - db = SQLitePartialExpandDbTesting.__new__(SQLitePartialExpandDbTesting) - db._db_handle = dbapi2.connect(path) # db is there but not yet init-ed - fail = True - self.assertRaises(Exception, db._ensure_schema) - fail = False - db._initialize(db._db_handle.cursor()) - - def test_open_database_non_existent(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/non-existent.sqlite' - self.assertRaises(errors.DatabaseDoesNotExist, - sqlcipher_open, path, "123", - create=False) - - def test_delete_database_existent(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/new.sqlite' - db = sqlcipher_open(path, "123", create=True) - db.close() - SQLCipherDatabase.delete_database(path) - self.assertRaises(errors.DatabaseDoesNotExist, - sqlcipher_open, path, "123", - create=False) - - def test_delete_database_nonexistent(self): - temp_dir = self.createTempDir(prefix='u1db-test-') - path = temp_dir + '/non-existent.sqlite' - self.assertRaises(errors.DatabaseDoesNotExist, - SQLCipherDatabase.delete_database, path) - - def test__get_indexed_fields(self): - self.db.create_index('idx1', 'a', 'b') - self.assertEqual(set(['a', 'b']), self.db._get_indexed_fields()) - self.db.create_index('idx2', 'b', 'c') - self.assertEqual(set(['a', 'b', 'c']), self.db._get_indexed_fields()) - - def test_indexed_fields_expanded(self): - self.db.create_index('idx1', 'key1') - doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') - self.assertEqual(set(['key1']), self.db._get_indexed_fields()) - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([(doc1.doc_id, 'key1', 'val1')], c.fetchall()) - - def test_create_index_updates_fields(self): - doc1 = self.db.create_doc_from_json('{"key1": "val1", "key2": "val2"}') - self.db.create_index('idx1', 'key1') - self.assertEqual(set(['key1']), self.db._get_indexed_fields()) - c = self.db._get_sqlite_handle().cursor() - c.execute("SELECT doc_id, field_name, value FROM document_fields" - " ORDER BY doc_id, field_name, value") - self.assertEqual([(doc1.doc_id, 'key1', 'val1')], c.fetchall()) - - def assertFormatQueryEquals(self, exp_statement, exp_args, definition, - values): - statement, args = self.db._format_query(definition, values) - self.assertEqual(exp_statement, statement) - self.assertEqual(exp_args, args) - - def test__format_query(self): - self.assertFormatQueryEquals( - "SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM " - "document d, document_fields d0 LEFT OUTER JOIN conflicts c ON " - "c.doc_id = d.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name " - "= ? AND d0.value = ? GROUP BY d.doc_id, d.doc_rev, d.content " - "ORDER BY d0.value;", ["key1", "a"], - ["key1"], ["a"]) - - def test__format_query2(self): - self.assertFormatQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value = ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value = ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value = ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' - 'd0.value, d1.value, d2.value;', - ["key1", "a", "key2", "b", "key3", "c"], - ["key1", "key2", "key3"], ["a", "b", "c"]) - - def test__format_query_wildcard(self): - self.assertFormatQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value = ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value GLOB ? AND d.doc_id = d2.doc_id AND d2.field_name = ? ' - 'AND d2.value NOT NULL GROUP BY d.doc_id, d.doc_rev, d.content ' - 'ORDER BY d0.value, d1.value, d2.value;', - ["key1", "a", "key2", "b*", "key3"], ["key1", "key2", "key3"], - ["a", "b*", "*"]) - - def assertFormatRangeQueryEquals(self, exp_statement, exp_args, definition, - start_value, end_value): - statement, args = self.db._format_range_query( - definition, start_value, end_value) - self.assertEqual(exp_statement, statement) - self.assertEqual(exp_args, args) - - def test__format_range_query(self): - self.assertFormatRangeQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value >= ? AND d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value <= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value <= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' - 'd0.value, d1.value, d2.value;', - ['key1', 'a', 'key2', 'b', 'key3', 'c', 'key1', 'p', 'key2', 'q', - 'key3', 'r'], - ["key1", "key2", "key3"], ["a", "b", "c"], ["p", "q", "r"]) - - def test__format_range_query_no_start(self): - self.assertFormatRangeQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value <= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value <= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' - 'd0.value, d1.value, d2.value;', - ['key1', 'a', 'key2', 'b', 'key3', 'c'], - ["key1", "key2", "key3"], None, ["a", "b", "c"]) - - def test__format_range_query_no_end(self): - self.assertFormatRangeQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value >= ? GROUP BY d.doc_id, d.doc_rev, d.content ORDER BY ' - 'd0.value, d1.value, d2.value;', - ['key1', 'a', 'key2', 'b', 'key3', 'c'], - ["key1", "key2", "key3"], ["a", "b", "c"], None) - - def test__format_range_query_wildcard(self): - self.assertFormatRangeQueryEquals( - 'SELECT d.doc_id, d.doc_rev, d.content, count(c.doc_rev) FROM ' - 'document d, document_fields d0, document_fields d1, ' - 'document_fields d2 LEFT OUTER JOIN conflicts c ON c.doc_id = ' - 'd.doc_id WHERE d.doc_id = d0.doc_id AND d0.field_name = ? AND ' - 'd0.value >= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? AND ' - 'd1.value >= ? AND d.doc_id = d2.doc_id AND d2.field_name = ? AND ' - 'd2.value NOT NULL AND d.doc_id = d0.doc_id AND d0.field_name = ? ' - 'AND d0.value <= ? AND d.doc_id = d1.doc_id AND d1.field_name = ? ' - 'AND (d1.value < ? OR d1.value GLOB ?) AND d.doc_id = d2.doc_id ' - 'AND d2.field_name = ? AND d2.value NOT NULL GROUP BY d.doc_id, ' - 'd.doc_rev, d.content ORDER BY d0.value, d1.value, d2.value;', - ['key1', 'a', 'key2', 'b', 'key3', 'key1', 'p', 'key2', 'q', 'q*', - 'key3'], - ["key1", "key2", "key3"], ["a", "b*", "*"], ["p", "q*", "*"]) - - -# ----------------------------------------------------------------------------- -# The following tests come from `u1db.tests.test_open`. -# ----------------------------------------------------------------------------- - - -class SQLCipherOpen(test_open.TestU1DBOpen): - - def test_open_no_create(self): - self.assertRaises(errors.DatabaseDoesNotExist, - sqlcipher_open, self.db_path, - PASSWORD, - create=False) - self.assertFalse(os.path.exists(self.db_path)) - - def test_open_create(self): - db = sqlcipher_open(self.db_path, PASSWORD, create=True) - self.addCleanup(db.close) - self.assertTrue(os.path.exists(self.db_path)) - self.assertIsInstance(db, SQLCipherDatabase) - - def test_open_with_factory(self): - db = sqlcipher_open(self.db_path, PASSWORD, create=True, - document_factory=SoledadDocument) - self.addCleanup(db.close) - doc = db.create_doc({}) - self.assertTrue(isinstance(doc, SoledadDocument)) - - def test_open_existing(self): - db = sqlcipher_open(self.db_path, PASSWORD) - self.addCleanup(db.close) - doc = db.create_doc_from_json(tests.simple_doc) - # Even though create=True, we shouldn't wipe the db - db2 = sqlcipher_open(self.db_path, PASSWORD, create=True) - self.addCleanup(db2.close) - doc2 = db2.get_doc(doc.doc_id) - self.assertEqual(doc, doc2) - - def test_open_existing_no_create(self): - db = sqlcipher_open(self.db_path, PASSWORD) - self.addCleanup(db.close) - db2 = sqlcipher_open(self.db_path, PASSWORD, create=False) - self.addCleanup(db2.close) - self.assertIsInstance(db2, SQLCipherDatabase) - - -# ----------------------------------------------------------------------------- -# Tests for actual encryption of the database -# ----------------------------------------------------------------------------- - -class SQLCipherEncryptionTests(BaseSoledadTest): - - """ - Tests to guarantee SQLCipher is indeed encrypting data when storing. - """ - - def _delete_dbfiles(self): - for dbfile in [self.DB_FILE]: - if os.path.exists(dbfile): - os.unlink(dbfile) - - def setUp(self): - # the following come from BaseLeapTest.setUpClass, because - # twisted.trial doesn't support such class methods for setting up - # test classes. - self.old_path = os.environ['PATH'] - self.old_home = os.environ['HOME'] - self.tempdir = tempfile.mkdtemp(prefix="leap_tests-") - self.home = self.tempdir - bin_tdir = os.path.join( - self.tempdir, - 'bin') - os.environ["PATH"] = bin_tdir - os.environ["HOME"] = self.tempdir - # this is our own stuff - self.DB_FILE = os.path.join(self.tempdir, 'test.db') - self._delete_dbfiles() - - def tearDown(self): - self._delete_dbfiles() - # the following come from BaseLeapTest.tearDownClass, because - # twisted.trial doesn't support such class methods for tearing down - # test classes. - os.environ["PATH"] = self.old_path - os.environ["HOME"] = self.old_home - # safety check! please do not wipe my home... - # XXX needs to adapt to non-linuces - soledad_assert( - self.tempdir.startswith('/tmp/leap_tests-') or - self.tempdir.startswith('/var/folder'), - "beware! tried to remove a dir which does not " - "live in temporal folder!") - shutil.rmtree(self.tempdir) - - def test_try_to_open_encrypted_db_with_sqlite_backend(self): - """ - SQLite backend should not succeed to open SQLCipher databases. - """ - db = sqlcipher_open(self.DB_FILE, PASSWORD) - doc = db.create_doc_from_json(tests.simple_doc) - db.close() - try: - # trying to open an encrypted database with the regular u1db - # backend should raise a DatabaseError exception. - SQLitePartialExpandDatabase(self.DB_FILE, - document_factory=SoledadDocument) - raise DatabaseIsNotEncrypted() - except dbapi2.DatabaseError: - # at this point we know that the regular U1DB sqlcipher backend - # did not succeed on opening the database, so it was indeed - # encrypted. - db = sqlcipher_open(self.DB_FILE, PASSWORD) - doc = db.get_doc(doc.doc_id) - self.assertEqual(tests.simple_doc, doc.get_json(), - 'decrypted content mismatch') - db.close() - - def test_try_to_open_raw_db_with_sqlcipher_backend(self): - """ - SQLCipher backend should not succeed to open unencrypted databases. - """ - db = SQLitePartialExpandDatabase(self.DB_FILE, - document_factory=SoledadDocument) - db.create_doc_from_json(tests.simple_doc) - db.close() - try: - # trying to open the a non-encrypted database with sqlcipher - # backend should raise a DatabaseIsNotEncrypted exception. - db = sqlcipher_open(self.DB_FILE, PASSWORD) - db.close() - raise dbapi2.DatabaseError( - "SQLCipher backend should not be able to open non-encrypted " - "dbs.") - except DatabaseIsNotEncrypted: - pass -- cgit v1.2.3 From 0bd98f5d6ab8548afb84a9570aee2d7dccd89db5 Mon Sep 17 00:00:00 2001 From: drebs Date: Mon, 11 Jul 2016 15:39:01 +0200 Subject: [test] use tox in gitlab-ci --- .gitlab-ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1aab6d74..647cc43c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,3 @@ -image: leap/soledad:1.0 - -test: +trial: script: - - /usr/local/soledad/run-trial-from-gitlab-ci.sh + - cd testing; tox -- cgit v1.2.3 From 9b0f6c70fd5b7adc507ea4a1d4cc80f4c05aa994 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 12 Jul 2016 03:13:55 +0200 Subject: [tests] ignore tox folder --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 0de6147f..6c3e413e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ MANIFEST *.log *.*~ *.csv +.cache +.tox .eggs _trial_temp .DS_Store -- cgit v1.2.3 From 179dfe95102f7314ca6a59a95dd1080080a5724a Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Tue, 12 Jul 2016 03:24:44 +0200 Subject: add pep8/flake8 to tox --- README.rst | 7 +++---- testing/tox.ini | 4 +++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 74a49d12..11455880 100644 --- a/README.rst +++ b/README.rst @@ -47,11 +47,10 @@ Compatibility Tests ----- -Client and server tests are both included in leap.soledad.common. If you want -to run tests in development mode you must do the following:: +Soledad tests use tox, and they live in the testing folder:: - scripts/develop_mode.sh - scripts/run_tests.sh + cd testing + tox Note that to run CouchDB tests, be sure you have `CouchDB`_ installed on your system. diff --git a/testing/tox.ini b/testing/tox.ini index 0a8dda9d..3663eef3 100644 --- a/testing/tox.ini +++ b/testing/tox.ini @@ -2,10 +2,12 @@ envlist = py27 [testenv] -commands = py.test {posargs} +commands = py.test --pep8 {posargs} changedir = tests deps = pytest + pytest-flake8 + pytest-pep8 mock testscenarios setuptools-trial -- cgit v1.2.3 From 7d264548d6df756f2a157fe59cf58b3240825418 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Wed, 13 Jul 2016 19:36:14 +0200 Subject: [style] pep8 --- .../src/leap/soledad/client/http_target/support.py | 1 + common/src/leap/soledad/common/l2db/__init__.py | 4 +- .../soledad/common/l2db/backends/sqlite_backend.py | 28 +- .../soledad/common/l2db/commandline/__init__.py | 15 - .../leap/soledad/common/l2db/commandline/client.py | 497 --------------------- .../soledad/common/l2db/commandline/command.py | 80 ---- .../leap/soledad/common/l2db/commandline/serve.py | 58 --- common/src/leap/soledad/common/l2db/errors.py | 3 +- .../src/leap/soledad/common/l2db/query_parser.py | 4 +- .../soledad/common/l2db/remote/cors_middleware.py | 42 -- .../leap/soledad/common/l2db/remote/http_app.py | 24 +- .../soledad/common/l2db/remote/http_database.py | 6 +- .../leap/soledad/common/l2db/remote/http_target.py | 2 +- .../soledad/common/l2db/remote/oauth_middleware.py | 89 ---- .../soledad/common/l2db/remote/server_state.py | 1 + .../common/l2db/remote/ssl_match_hostname.py | 11 +- common/src/leap/soledad/common/l2db/sync.py | 32 +- 17 files changed, 63 insertions(+), 834 deletions(-) delete mode 100644 common/src/leap/soledad/common/l2db/commandline/__init__.py delete mode 100644 common/src/leap/soledad/common/l2db/commandline/client.py delete mode 100644 common/src/leap/soledad/common/l2db/commandline/command.py delete mode 100644 common/src/leap/soledad/common/l2db/commandline/serve.py delete mode 100644 common/src/leap/soledad/common/l2db/remote/cors_middleware.py delete mode 100644 common/src/leap/soledad/common/l2db/remote/oauth_middleware.py diff --git a/client/src/leap/soledad/client/http_target/support.py b/client/src/leap/soledad/client/http_target/support.py index d82fe346..6ec98ed4 100644 --- a/client/src/leap/soledad/client/http_target/support.py +++ b/client/src/leap/soledad/client/http_target/support.py @@ -31,6 +31,7 @@ from leap.soledad.common.l2db.remote import http_errors # twisted. Because of that, we redefine the http body reader used by the HTTP # client below. + class ReadBodyProtocol(_ReadBodyProtocol): """ From original Twisted implementation, focused on adding our error diff --git a/common/src/leap/soledad/common/l2db/__init__.py b/common/src/leap/soledad/common/l2db/__init__.py index cc121d06..c0bd15fe 100644 --- a/common/src/leap/soledad/common/l2db/__init__.py +++ b/common/src/leap/soledad/common/l2db/__init__.py @@ -464,8 +464,8 @@ class DocumentBase(object): """ # Since this is just for testing, we don't worry about comparing # against things that aren't a Document. - return ((self.doc_id, self.rev, self.get_json()) - < (other.doc_id, other.rev, other.get_json())) + return ((self.doc_id, self.rev, self.get_json()) < + (other.doc_id, other.rev, other.get_json())) def get_json(self): """Get the json serialization of this document.""" diff --git a/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py b/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py index 309000ee..ba273039 100644 --- a/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py +++ b/common/src/leap/soledad/common/l2db/backends/sqlite_backend.py @@ -156,7 +156,7 @@ class SQLiteDatabase(CommonBackend): def _initialize(self, c): """Create the schema in the database.""" - #read the script with sql commands + # read the script with sql commands # TODO: Change how we set up the dependency. Most likely use something # like lp:dirspec to grab the file from a common resource # directory. Doesn't specifically need to be handled until we get @@ -172,7 +172,7 @@ class SQLiteDatabase(CommonBackend): if not line: continue c.execute(line) - #add extra fields + # add extra fields self._extra_schema_init(c) # A unique identifier should be set for this replica. Implementations # don't have to strictly use uuid here, but we do want the uid to be @@ -509,7 +509,8 @@ class SQLiteDatabase(CommonBackend): def _put_doc_if_newer(self, doc, save_conflict, replica_uid=None, replica_gen=None, replica_trans_id=None): with self._db_handle: - return super(SQLiteDatabase, self)._put_doc_if_newer(doc, + return super(SQLiteDatabase, self)._put_doc_if_newer( + doc, save_conflict=save_conflict, replica_uid=replica_uid, replica_gen=replica_gen, replica_trans_id=replica_trans_id) @@ -620,14 +621,14 @@ class SQLiteDatabase(CommonBackend): novalue_where = ["d.doc_id = d%d.doc_id" " AND d%d.field_name = ?" % (i, i) for i in range(len(definition))] - wildcard_where = [novalue_where[i] - + (" AND d%d.value NOT NULL" % (i,)) + wildcard_where = [novalue_where[i] + + (" AND d%d.value NOT NULL" % (i,)) for i in range(len(definition))] - exact_where = [novalue_where[i] - + (" AND d%d.value = ?" % (i,)) + exact_where = [novalue_where[i] + + (" AND d%d.value = ?" % (i,)) for i in range(len(definition))] - like_where = [novalue_where[i] - + (" AND d%d.value GLOB ?" % (i,)) + like_where = [novalue_where[i] + + (" AND d%d.value GLOB ?" % (i,)) for i in range(len(definition))] is_wildcard = False # Merge the lists together, so that: @@ -672,7 +673,8 @@ class SQLiteDatabase(CommonBackend): try: c.execute(statement, tuple(args)) except dbapi2.OperationalError, e: - raise dbapi2.OperationalError(str(e) + + raise dbapi2.OperationalError( + str(e) + '\nstatement: %s\nargs: %s\n' % (statement, args)) res = c.fetchall() results = [] @@ -771,7 +773,8 @@ class SQLiteDatabase(CommonBackend): try: c.execute(statement, tuple(args)) except dbapi2.OperationalError, e: - raise dbapi2.OperationalError(str(e) + + raise dbapi2.OperationalError( + str(e) + '\nstatement: %s\nargs: %s\n' % (statement, args)) res = c.fetchall() results = [] @@ -800,7 +803,8 @@ class SQLiteDatabase(CommonBackend): try: c.execute(statement, tuple(definition)) except dbapi2.OperationalError, e: - raise dbapi2.OperationalError(str(e) + + raise dbapi2.OperationalError( + str(e) + '\nstatement: %s\nargs: %s\n' % (statement, tuple(definition))) return c.fetchall() diff --git a/common/src/leap/soledad/common/l2db/commandline/__init__.py b/common/src/leap/soledad/common/l2db/commandline/__init__.py deleted file mode 100644 index 3f32e381..00000000 --- a/common/src/leap/soledad/common/l2db/commandline/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see . diff --git a/common/src/leap/soledad/common/l2db/commandline/client.py b/common/src/leap/soledad/common/l2db/commandline/client.py deleted file mode 100644 index 15bf8561..00000000 --- a/common/src/leap/soledad/common/l2db/commandline/client.py +++ /dev/null @@ -1,497 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see . - -"""Commandline bindings for the u1db-client program.""" - -import argparse -import os -try: - import simplejson as json -except ImportError: - import json # noqa -import sys - -from u1db import ( - Document, - open as u1db_open, - sync, - errors, - ) -from u1db.commandline import command -from u1db.remote import ( - http_database, - http_target, - ) - - -client_commands = command.CommandGroup() - - -def set_oauth_credentials(client): - keys = os.environ.get('OAUTH_CREDENTIALS', None) - if keys is not None: - consumer_key, consumer_secret, \ - token_key, token_secret = keys.split(":") - client.set_oauth_credentials(consumer_key, consumer_secret, - token_key, token_secret) - - -class OneDbCmd(command.Command): - """Base class for commands operating on one local or remote database.""" - - def _open(self, database, create): - if database.startswith(('http://', 'https://')): - db = http_database.HTTPDatabase(database) - set_oauth_credentials(db) - db.open(create) - return db - else: - return u1db_open(database, create) - - -class CmdCreate(OneDbCmd): - """Create a new document from scratch""" - - name = 'create' - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', - help='The local or remote database to update', - metavar='database-path-or-url') - parser.add_argument('infile', nargs='?', default=None, - help='The file to read content from.') - parser.add_argument('--id', dest='doc_id', default=None, - help='Set the document identifier') - - def run(self, database, infile, doc_id): - if infile is None: - infile = self.stdin - db = self._open(database, create=False) - doc = db.create_doc_from_json(infile.read(), doc_id=doc_id) - self.stderr.write('id: %s\nrev: %s\n' % (doc.doc_id, doc.rev)) - -client_commands.register(CmdCreate) - - -class CmdDelete(OneDbCmd): - """Delete a document from the database""" - - name = 'delete' - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', - help='The local or remote database to update', - metavar='database-path-or-url') - parser.add_argument('doc_id', help='The document id to retrieve') - parser.add_argument('doc_rev', - help='The revision of the document (which is being superseded.)') - - def run(self, database, doc_id, doc_rev): - db = self._open(database, create=False) - doc = Document(doc_id, doc_rev, None) - db.delete_doc(doc) - self.stderr.write('rev: %s\n' % (doc.rev,)) - -client_commands.register(CmdDelete) - - -class CmdGet(OneDbCmd): - """Extract a document from the database""" - - name = 'get' - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', - help='The local or remote database to query', - metavar='database-path-or-url') - parser.add_argument('doc_id', help='The document id to retrieve.') - parser.add_argument('outfile', nargs='?', default=None, - help='The file to write the document to', - type=argparse.FileType('wb')) - - def run(self, database, doc_id, outfile): - if outfile is None: - outfile = self.stdout - try: - db = self._open(database, create=False) - except errors.DatabaseDoesNotExist: - self.stderr.write("Database does not exist.\n") - return 1 - doc = db.get_doc(doc_id) - if doc is None: - self.stderr.write('Document not found (id: %s)\n' % (doc_id,)) - return 1 # failed - if doc.is_tombstone(): - outfile.write('[document deleted]\n') - else: - outfile.write(doc.get_json() + '\n') - self.stderr.write('rev: %s\n' % (doc.rev,)) - if doc.has_conflicts: - self.stderr.write("Document has conflicts.\n") - -client_commands.register(CmdGet) - - -class CmdGetDocConflicts(OneDbCmd): - """Get the conflicts from a document""" - - name = 'get-doc-conflicts' - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', - help='The local database to query', - metavar='database-path') - parser.add_argument('doc_id', help='The document id to retrieve.') - - def run(self, database, doc_id): - try: - db = self._open(database, False) - except errors.DatabaseDoesNotExist: - self.stderr.write("Database does not exist.\n") - return 1 - conflicts = db.get_doc_conflicts(doc_id) - if not conflicts: - if db.get_doc(doc_id) is None: - self.stderr.write("Document does not exist.\n") - return 1 - self.stdout.write("[") - for i, doc in enumerate(conflicts): - if i: - self.stdout.write(",") - self.stdout.write( - json.dumps(dict(rev=doc.rev, content=doc.content), indent=4)) - self.stdout.write("]\n") - -client_commands.register(CmdGetDocConflicts) - - -class CmdInitDB(OneDbCmd): - """Create a new database""" - - name = 'init-db' - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', - help='The local or remote database to create', - metavar='database-path-or-url') - parser.add_argument('--replica-uid', default=None, - help='The unique identifier for this database (not for remote)') - - def run(self, database, replica_uid): - db = self._open(database, create=True) - if replica_uid is not None: - db._set_replica_uid(replica_uid) - -client_commands.register(CmdInitDB) - - -class CmdPut(OneDbCmd): - """Add a document to the database""" - - name = 'put' - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', - help='The local or remote database to update', - metavar='database-path-or-url'), - parser.add_argument('doc_id', help='The document id to retrieve') - parser.add_argument('doc_rev', - help='The revision of the document (which is being superseded.)') - parser.add_argument('infile', nargs='?', default=None, - help='The filename of the document that will be used for content', - type=argparse.FileType('rb')) - - def run(self, database, doc_id, doc_rev, infile): - if infile is None: - infile = self.stdin - try: - db = self._open(database, create=False) - doc = Document(doc_id, doc_rev, infile.read()) - doc_rev = db.put_doc(doc) - self.stderr.write('rev: %s\n' % (doc_rev,)) - except errors.DatabaseDoesNotExist: - self.stderr.write("Database does not exist.\n") - except errors.RevisionConflict: - if db.get_doc(doc_id) is None: - self.stderr.write("Document does not exist.\n") - else: - self.stderr.write("Given revision is not current.\n") - except errors.ConflictedDoc: - self.stderr.write( - "Document has conflicts.\n" - "Inspect with get-doc-conflicts, then resolve.\n") - else: - return - return 1 - -client_commands.register(CmdPut) - - -class CmdResolve(OneDbCmd): - """Resolve a conflicted document""" - - name = 'resolve-doc' - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', - help='The local or remote database to update', - metavar='database-path-or-url'), - parser.add_argument('doc_id', help='The conflicted document id') - parser.add_argument('doc_revs', metavar="doc-rev", nargs="+", - help='The revisions that the new content supersedes') - parser.add_argument('--infile', nargs='?', default=None, - help='The filename of the document that will be used for content', - type=argparse.FileType('rb')) - - def run(self, database, doc_id, doc_revs, infile): - if infile is None: - infile = self.stdin - try: - db = self._open(database, create=False) - except errors.DatabaseDoesNotExist: - self.stderr.write("Database does not exist.\n") - return 1 - doc = db.get_doc(doc_id) - if doc is None: - self.stderr.write("Document does not exist.\n") - return 1 - doc.set_json(infile.read()) - db.resolve_doc(doc, doc_revs) - self.stderr.write("rev: %s\n" % db.get_doc(doc_id).rev) - if doc.has_conflicts: - self.stderr.write("Document still has conflicts.\n") - -client_commands.register(CmdResolve) - - -class CmdSync(command.Command): - """Synchronize two databases""" - - name = 'sync' - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('source', help='database to sync from') - parser.add_argument('target', help='database to sync to') - - def _open_target(self, target): - if target.startswith(('http://', 'https://')): - st = http_target.HTTPSyncTarget.connect(target) - set_oauth_credentials(st) - else: - db = u1db_open(target, create=True) - st = db.get_sync_target() - return st - - def run(self, source, target): - """Start a Sync request.""" - source_db = u1db_open(source, create=False) - st = self._open_target(target) - syncer = sync.Synchronizer(source_db, st) - syncer.sync() - source_db.close() - -client_commands.register(CmdSync) - - -class CmdCreateIndex(OneDbCmd): - """Create an index""" - - name = "create-index" - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', help='The local database to update', - metavar='database-path') - parser.add_argument('index', help='the name of the index') - parser.add_argument('expression', help='an index expression', - nargs='+') - - def run(self, database, index, expression): - try: - db = self._open(database, create=False) - db.create_index(index, *expression) - except errors.DatabaseDoesNotExist: - self.stderr.write("Database does not exist.\n") - return 1 - except errors.IndexNameTakenError: - self.stderr.write("There is already a different index named %r.\n" - % (index,)) - return 1 - except errors.IndexDefinitionParseError: - self.stderr.write("Bad index expression.\n") - return 1 - -client_commands.register(CmdCreateIndex) - - -class CmdListIndexes(OneDbCmd): - """List existing indexes""" - - name = "list-indexes" - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', help='The local database to query', - metavar='database-path') - - def run(self, database): - try: - db = self._open(database, create=False) - except errors.DatabaseDoesNotExist: - self.stderr.write("Database does not exist.\n") - return 1 - for (index, expression) in db.list_indexes(): - self.stdout.write("%s: %s\n" % (index, ", ".join(expression))) - -client_commands.register(CmdListIndexes) - - -class CmdDeleteIndex(OneDbCmd): - """Delete an index""" - - name = "delete-index" - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', help='The local database to update', - metavar='database-path') - parser.add_argument('index', help='the name of the index') - - def run(self, database, index): - try: - db = self._open(database, create=False) - except errors.DatabaseDoesNotExist: - self.stderr.write("Database does not exist.\n") - return 1 - db.delete_index(index) - -client_commands.register(CmdDeleteIndex) - - -class CmdGetIndexKeys(OneDbCmd): - """Get the index's keys""" - - name = "get-index-keys" - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', help='The local database to query', - metavar='database-path') - parser.add_argument('index', help='the name of the index') - - def run(self, database, index): - try: - db = self._open(database, create=False) - for key in db.get_index_keys(index): - self.stdout.write("%s\n" % (", ".join( - [i.encode('utf-8') for i in key],))) - except errors.DatabaseDoesNotExist: - self.stderr.write("Database does not exist.\n") - except errors.IndexDoesNotExist: - self.stderr.write("Index does not exist.\n") - else: - return - return 1 - -client_commands.register(CmdGetIndexKeys) - - -class CmdGetFromIndex(OneDbCmd): - """Find documents by searching an index""" - - name = "get-from-index" - argv = None - - @classmethod - def _populate_subparser(cls, parser): - parser.add_argument('database', help='The local database to query', - metavar='database-path') - parser.add_argument('index', help='the name of the index') - parser.add_argument('values', metavar="value", - help='the value to look up (one per index column)', - nargs="+") - - def run(self, database, index, values): - try: - db = self._open(database, create=False) - docs = db.get_from_index(index, *values) - except errors.DatabaseDoesNotExist: - self.stderr.write("Database does not exist.\n") - except errors.IndexDoesNotExist: - self.stderr.write("Index does not exist.\n") - except errors.InvalidValueForIndex: - index_def = db._get_index_definition(index) - len_diff = len(index_def) - len(values) - if len_diff == 0: - # can't happen (HAH) - raise - argv = self.argv if self.argv is not None else sys.argv - self.stderr.write( - "Invalid query: " - "index %r requires %d query expression%s%s.\n" - "For example, the following would be valid:\n" - " %s %s %r %r %s\n" - % (index, - len(index_def), - "s" if len(index_def) > 1 else "", - ", not %d" % len(values) if len(values) else "", - argv[0], argv[1], database, index, - " ".join(map(repr, - values[:len(index_def)] - + ["*" for i in range(len_diff)])), - )) - except errors.InvalidGlobbing: - argv = self.argv if self.argv is not None else sys.argv - fixed = [] - for (i, v) in enumerate(values): - fixed.append(v) - if v.endswith('*'): - break - # values has at least one element, so i is defined - fixed.extend('*' * (len(values) - i - 1)) - self.stderr.write( - "Invalid query: a star can only be followed by stars.\n" - "For example, the following would be valid:\n" - " %s %s %r %r %s\n" - % (argv[0], argv[1], database, index, - " ".join(map(repr, fixed)))) - - else: - self.stdout.write("[") - for i, doc in enumerate(docs): - if i: - self.stdout.write(",") - self.stdout.write( - json.dumps( - dict(id=doc.doc_id, rev=doc.rev, content=doc.content), - indent=4)) - self.stdout.write("]\n") - return - return 1 - -client_commands.register(CmdGetFromIndex) - - -def main(args): - return client_commands.run_argv(args, sys.stdin, sys.stdout, sys.stderr) diff --git a/common/src/leap/soledad/common/l2db/commandline/command.py b/common/src/leap/soledad/common/l2db/commandline/command.py deleted file mode 100644 index eace0560..00000000 --- a/common/src/leap/soledad/common/l2db/commandline/command.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see . - -"""Command infrastructure for u1db""" - -import argparse -import inspect - - -class CommandGroup(object): - """A collection of commands.""" - - def __init__(self, description=None): - self.commands = {} - self.description = description - - def register(self, cmd): - """Register a new command to be incorporated with this group.""" - self.commands[cmd.name] = cmd - - def make_argparser(self): - """Create an argparse.ArgumentParser""" - parser = argparse.ArgumentParser(description=self.description) - subs = parser.add_subparsers(title='commands') - for name, cmd in sorted(self.commands.iteritems()): - sub = subs.add_parser(name, help=cmd.__doc__) - sub.set_defaults(subcommand=cmd) - cmd._populate_subparser(sub) - return parser - - def run_argv(self, argv, stdin, stdout, stderr): - """Run a command, from a sys.argv[1:] style input.""" - parser = self.make_argparser() - args = parser.parse_args(argv) - cmd = args.subcommand(stdin, stdout, stderr) - params, _, _, _ = inspect.getargspec(cmd.run) - vals = [] - for param in params[1:]: - vals.append(getattr(args, param)) - return cmd.run(*vals) - - -class Command(object): - """Definition of a Command that can be run. - - :cvar name: The name of the command, so that you can run - 'u1db-client '. - """ - - name = None - - def __init__(self, stdin, stdout, stderr): - self.stdin = stdin - self.stdout = stdout - self.stderr = stderr - - @classmethod - def _populate_subparser(cls, parser): - """Child classes should override this to provide their arguments.""" - raise NotImplementedError(cls._populate_subparser) - - def run(self, *args): - """This is where the magic happens. - - Subclasses should implement this, requesting their specific arguments. - """ - raise NotImplementedError(self.run) diff --git a/common/src/leap/soledad/common/l2db/commandline/serve.py b/common/src/leap/soledad/common/l2db/commandline/serve.py deleted file mode 100644 index 5e10f9cb..00000000 --- a/common/src/leap/soledad/common/l2db/commandline/serve.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2011 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see . - -"""Build server for u1db-serve.""" -import os - -from paste import httpserver - -from u1db.remote import ( - http_app, - server_state, - cors_middleware - ) - - -class DbListingServerState(server_state.ServerState): - """ServerState capable of listing dbs.""" - - def global_info(self): - """Return list of dbs.""" - dbs = [] - for fname in os.listdir(self._workingdir): - p = os.path.join(self._workingdir, fname) - if os.path.isfile(p) and os.access(p, os.R_OK|os.W_OK): - try: - with open(p, 'rb') as f: - header = f.read(16) - if header == "SQLite format 3\000": - dbs.append(fname) - except IOError: - pass - return {"databases": dict.fromkeys(dbs), "db_count": len(dbs)} - - -def make_server(host, port, working_dir, accept_cors_connections=None): - """Make a server on host and port exposing dbs living in working_dir.""" - state = DbListingServerState() - state.set_workingdir(working_dir) - application = http_app.HTTPApp(state) - if accept_cors_connections: - application = cors_middleware.CORSMiddleware(application, - accept_cors_connections) - server = httpserver.WSGIServer(application, (host, port), - httpserver.WSGIHandler) - return server diff --git a/common/src/leap/soledad/common/l2db/errors.py b/common/src/leap/soledad/common/l2db/errors.py index e5ee8f45..b502fc2d 100644 --- a/common/src/leap/soledad/common/l2db/errors.py +++ b/common/src/leap/soledad/common/l2db/errors.py @@ -185,8 +185,7 @@ class UnknownAuthMethod(U1DBError): # mapping wire (transimission) descriptions/tags for errors to the exceptions wire_description_to_exc = dict( (x.wire_description, x) for x in globals().values() - if getattr(x, 'wire_description', None) not in (None, "error") -) + if getattr(x, 'wire_description', None) not in (None, "error")) wire_description_to_exc["error"] = U1DBError diff --git a/common/src/leap/soledad/common/l2db/query_parser.py b/common/src/leap/soledad/common/l2db/query_parser.py index 7f07b554..dd35b12a 100644 --- a/common/src/leap/soledad/common/l2db/query_parser.py +++ b/common/src/leap/soledad/common/l2db/query_parser.py @@ -358,8 +358,8 @@ class Parser(object): @classmethod def register_transormation(cls, transform): assert transform.name not in cls._transformations, ( - "Transform %s already registered for %s" - % (transform.name, cls._transformations[transform.name])) + "Transform %s already registered for %s" + % (transform.name, cls._transformations[transform.name])) cls._transformations[transform.name] = transform diff --git a/common/src/leap/soledad/common/l2db/remote/cors_middleware.py b/common/src/leap/soledad/common/l2db/remote/cors_middleware.py deleted file mode 100644 index 8041b968..00000000 --- a/common/src/leap/soledad/common/l2db/remote/cors_middleware.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2012 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see . -"""U1DB Cross-Origin Resource Sharing WSGI middleware.""" - - -class CORSMiddleware(object): - """U1DB Cross-Origin Resource Sharing WSGI middleware.""" - - def __init__(self, app, accept_cors_connections): - self.origins = ' '.join(accept_cors_connections) - self.app = app - - def _cors_headers(self): - return [('access-control-allow-origin', self.origins), - ('access-control-allow-headers', - 'authorization, content-type, x-requested-with'), - ('access-control-allow-methods', - 'GET, POST, PUT, DELETE, OPTIONS')] - - def __call__(self, environ, start_response): - def wrap_start_response(status, headers, exc_info=None): - headers += self._cors_headers() - return start_response(status, headers, exc_info) - - if environ['REQUEST_METHOD'].lower() == 'options': - wrap_start_response("200 OK", [('content-type', 'text/plain')]) - return [''] - - return self.app(environ, wrap_start_response) diff --git a/common/src/leap/soledad/common/l2db/remote/http_app.py b/common/src/leap/soledad/common/l2db/remote/http_app.py index 3b65f5f7..65277bd1 100644 --- a/common/src/leap/soledad/common/l2db/remote/http_app.py +++ b/common/src/leap/soledad/common/l2db/remote/http_app.py @@ -18,7 +18,6 @@ """ HTTP Application exposing U1DB. """ - # TODO -- deprecate, use twisted/txaio. import functools @@ -340,18 +339,18 @@ class DocResource(object): headers={ 'x-u1db-rev': '', 'x-u1db-has-conflicts': 'false' - }) + }) return headers = { 'x-u1db-rev': doc.rev, 'x-u1db-has-conflicts': json.dumps(doc.has_conflicts) - } + } if doc.is_tombstone(): self.responder.send_response_json( - http_errors.wire_description_to_status[ - errors.DOCUMENT_DELETED], - error=errors.DOCUMENT_DELETED, - headers=headers) + http_errors.wire_description_to_status[ + errors.DOCUMENT_DELETED], + error=errors.DOCUMENT_DELETED, + headers=headers) else: self.responder.send_response_content( doc.get_json(), headers=headers) @@ -431,7 +430,7 @@ class SyncResource(object): self.responder.start_response(200) self.responder.start_stream(), header = {"new_generation": new_gen, - "new_transaction_id": self.sync_exch.new_trans_id} + "new_transaction_id": self.sync_exch.new_trans_id} if self.replica_uid is not None: header['replica_uid'] = self.replica_uid self.responder.stream_entry(header) @@ -462,10 +461,11 @@ class HTTPResponder(object): return self._started = True status_text = httplib.responses[status] - self._write = self._start_response('%d %s' % (status, status_text), - [('content-type', self.content_type), - ('cache-control', 'no-cache')] + - headers.items()) + self._write = self._start_response( + '%d %s' % (status, status_text), + [('content-type', self.content_type), + ('cache-control', 'no-cache')] + + headers.items()) # xxx version in headers if obj_dic is not None: self._no_initial_obj = False diff --git a/common/src/leap/soledad/common/l2db/remote/http_database.py b/common/src/leap/soledad/common/l2db/remote/http_database.py index d8dcfd55..b2b48dee 100644 --- a/common/src/leap/soledad/common/l2db/remote/http_database.py +++ b/common/src/leap/soledad/common/l2db/remote/http_database.py @@ -92,9 +92,9 @@ class HTTPDatabase(http_client.HTTPClientBase, Database): return None except errors.HTTPError, e: if (e.status == DOCUMENT_DELETED_STATUS and - 'x-u1db-rev' in e.headers): - res = None - headers = e.headers + 'x-u1db-rev' in e.headers): + res = None + headers = e.headers else: raise doc_rev = headers['x-u1db-rev'] diff --git a/common/src/leap/soledad/common/l2db/remote/http_target.py b/common/src/leap/soledad/common/l2db/remote/http_target.py index 598170e4..7e7f366f 100644 --- a/common/src/leap/soledad/common/l2db/remote/http_target.py +++ b/common/src/leap/soledad/common/l2db/remote/http_target.py @@ -47,7 +47,7 @@ class HTTPSyncTarget(http_client.HTTPClientBase, SyncTarget): if self._trace_hook: # for tests self._trace_hook('record_sync_info') self._request_json('PUT', ['sync-from', source_replica_uid], {}, - {'generation': source_replica_generation, + {'generation': source_replica_generation, 'transaction_id': source_transaction_id}) def _parse_sync_stream(self, data, return_doc_cb, ensure_callback=None): diff --git a/common/src/leap/soledad/common/l2db/remote/oauth_middleware.py b/common/src/leap/soledad/common/l2db/remote/oauth_middleware.py deleted file mode 100644 index 5772580a..00000000 --- a/common/src/leap/soledad/common/l2db/remote/oauth_middleware.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2012 Canonical Ltd. -# -# This file is part of u1db. -# -# u1db is free software: you can redistribute it and/or modify -# it under the terms of the GNU Lesser General Public License version 3 -# as published by the Free Software Foundation. -# -# u1db is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with u1db. If not, see . -"""U1DB OAuth authorisation WSGI middleware.""" -import httplib -from oauth import oauth -try: - import simplejson as json -except ImportError: - import json # noqa -from urllib import quote -from wsgiref.util import shift_path_info - - -sign_meth_HMAC_SHA1 = oauth.OAuthSignatureMethod_HMAC_SHA1() -sign_meth_PLAINTEXT = oauth.OAuthSignatureMethod_PLAINTEXT() - - -class OAuthMiddleware(object): - """U1DB OAuth Authorisation WSGI middleware.""" - - # max seconds the request timestamp is allowed to be shifted - # from arrival time - timestamp_threshold = 300 - - def __init__(self, app, base_url, prefix='/~/'): - self.app = app - self.base_url = base_url - self.prefix = prefix - - def get_oauth_data_store(self): - """Provide a oauth.OAuthDataStore.""" - raise NotImplementedError(self.get_oauth_data_store) - - def _error(self, start_response, status, description, message=None): - start_response("%d %s" % (status, httplib.responses[status]), - [('content-type', 'application/json')]) - err = {"error": description} - if message: - err['message'] = message - return [json.dumps(err)] - - def __call__(self, environ, start_response): - if self.prefix and not environ['PATH_INFO'].startswith(self.prefix): - return self._error(start_response, 400, "bad request") - headers = {} - if 'HTTP_AUTHORIZATION' in environ: - headers['Authorization'] = environ['HTTP_AUTHORIZATION'] - oauth_req = oauth.OAuthRequest.from_request( - http_method=environ['REQUEST_METHOD'], - http_url=self.base_url + environ['PATH_INFO'], - headers=headers, - query_string=environ['QUERY_STRING'] - ) - if oauth_req is None: - return self._error(start_response, 401, "unauthorized", - "Missing OAuth.") - try: - self.verify(environ, oauth_req) - except oauth.OAuthError, e: - return self._error(start_response, 401, "unauthorized", - e.message) - shift_path_info(environ) - return self.app(environ, start_response) - - def verify(self, environ, oauth_req): - """Verify OAuth request, put user_id in the environ.""" - oauth_server = oauth.OAuthServer(self.get_oauth_data_store()) - oauth_server.timestamp_threshold = self.timestamp_threshold - oauth_server.add_signature_method(sign_meth_HMAC_SHA1) - oauth_server.add_signature_method(sign_meth_PLAINTEXT) - consumer, token, parameters = oauth_server.verify_request(oauth_req) - # filter out oauth bits - environ['QUERY_STRING'] = '&'.join("%s=%s" % (quote(k, safe=''), - quote(v, safe='')) - for k, v in parameters.iteritems()) - return consumer, token diff --git a/common/src/leap/soledad/common/l2db/remote/server_state.py b/common/src/leap/soledad/common/l2db/remote/server_state.py index 6c1104c6..f131e09e 100644 --- a/common/src/leap/soledad/common/l2db/remote/server_state.py +++ b/common/src/leap/soledad/common/l2db/remote/server_state.py @@ -18,6 +18,7 @@ import os import errno + class ServerState(object): """Passed to a Request when it is instantiated. diff --git a/common/src/leap/soledad/common/l2db/remote/ssl_match_hostname.py b/common/src/leap/soledad/common/l2db/remote/ssl_match_hostname.py index fbabc177..ce82f1b2 100644 --- a/common/src/leap/soledad/common/l2db/remote/ssl_match_hostname.py +++ b/common/src/leap/soledad/common/l2db/remote/ssl_match_hostname.py @@ -52,13 +52,14 @@ def match_hostname(cert, hostname): return dnsnames.append(value) if len(dnsnames) > 1: - raise CertificateError("hostname %r " - "doesn't match either of %s" + raise CertificateError( + "hostname %r doesn't match either of %s" % (hostname, ', '.join(map(repr, dnsnames)))) elif len(dnsnames) == 1: - raise CertificateError("hostname %r " - "doesn't match %r" + raise CertificateError( + "hostname %r doesn't match %r" % (hostname, dnsnames[0])) else: - raise CertificateError("no appropriate commonName or " + raise CertificateError( + "no appropriate commonName or " "subjectAltName fields were found") diff --git a/common/src/leap/soledad/common/l2db/sync.py b/common/src/leap/soledad/common/l2db/sync.py index 26e67140..c612629f 100644 --- a/common/src/leap/soledad/common/l2db/sync.py +++ b/common/src/leap/soledad/common/l2db/sync.py @@ -53,7 +53,8 @@ class Synchronizer(object): """ # Increases self.num_inserted depending whether the document # was effectively inserted. - state, _ = self.source._put_doc_if_newer(doc, save_conflict=True, + state, _ = self.source._put_doc_if_newer( + doc, save_conflict=True, replica_uid=self.target_replica_uid, replica_gen=replica_gen, replica_trans_id=trans_id) if state == 'inserted': @@ -85,10 +86,10 @@ class Synchronizer(object): new generation. """ cur_gen, trans_id = self.source._get_generation_info() - if (cur_gen == start_generation + self.num_inserted - and self.num_inserted > 0): - self.sync_target.record_sync_info( - self.source._replica_uid, cur_gen, trans_id) + last_gen = start_generation + self.num_inserted + if (cur_gen == last_gen and self.num_inserted > 0): + self.sync_target.record_sync_info( + self.source._replica_uid, cur_gen, trans_id) def sync(self, callback=None, autocreate=False): """Synchronize documents between source and target.""" @@ -124,15 +125,17 @@ class Synchronizer(object): if self.target_replica_uid is None: target_last_known_gen, target_last_known_trans_id = 0, '' else: - target_last_known_gen, target_last_known_trans_id = \ - self.source._get_replica_gen_and_trans_id(self.target_replica_uid) + target_last_known_gen, target_last_known_trans_id = ( + self.source._get_replica_gen_and_trans_id( # nopep8 + self.target_replica_uid)) if not changes and target_last_known_gen == target_gen: if target_trans_id != target_last_known_trans_id: raise errors.InvalidTransactionId return my_gen changed_doc_ids = [doc_id for doc_id, _, _ in changes] # prepare to send all the changed docs - docs_to_send = self.source.get_docs(changed_doc_ids, + docs_to_send = self.source.get_docs( + changed_doc_ids, check_for_conflicts=False, include_deleted=True) # TODO: there must be a way to not iterate twice docs_by_generation = zip( @@ -172,7 +175,7 @@ class SyncExchange(object): self._db._last_exchange_log = { 'receive': {'docs': self._incoming_trace}, 'return': None - } + } def _set_trace_hook(self, cb): self._trace_hook = cb @@ -198,7 +201,8 @@ class SyncExchange(object): :param source_gen: The source generation of doc. :return: None """ - state, at_gen = self._db._put_doc_if_newer(doc, save_conflict=False, + state, at_gen = self._db._put_doc_if_newer( + doc, save_conflict=False, replica_uid=self.source_replica_uid, replica_gen=source_gen, replica_trans_id=trans_id) if state == 'inserted': @@ -217,7 +221,7 @@ class SyncExchange(object): self._db._last_exchange_log['receive'].update({ 'source_uid': self.source_replica_uid, 'source_gen': source_gen - }) + }) def find_changes_to_return(self): """Find changes to return. @@ -232,7 +236,7 @@ class SyncExchange(object): """ self._db._last_exchange_log['receive'].update({ # for tests 'last_known_gen': self.source_last_known_generation - }) + }) self._trace('before whats_changed') gen, trans_id, changes = self._db.whats_changed( self.source_last_known_generation) @@ -242,9 +246,9 @@ class SyncExchange(object): seen_ids = self.seen_ids # changed docs that weren't superseded by or converged with self.changes_to_return = [ - (doc_id, gen, trans_id) for (doc_id, gen, trans_id) in changes + (doc_id, gen, trans_id) for (doc_id, gen, trans_id) in changes if # there was a subsequent update - if doc_id not in seen_ids or seen_ids.get(doc_id) < gen] + doc_id not in seen_ids or seen_ids.get(doc_id) < gen] return self.new_gen def return_docs(self, return_doc_cb): -- cgit v1.2.3 From 6b23b3f3215f2443aa3e790559b63a41b3040072 Mon Sep 17 00:00:00 2001 From: Kali Kaneko Date: Thu, 14 Jul 2016 10:19:21 +0200 Subject: [pkg] bump changelog to 0.8.1 --- CHANGELOG.rst | 37 +++++++++++++++++++++++++++++++++++++ client/changes/next-changelog.rst | 13 +------------ common/changes/next-changelog.rst | 2 +- server/changes/next-changelog.rst | 2 +- 4 files changed, 40 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ed60b5ec..24c20641 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,3 +1,40 @@ +0.8.1 - 14 July, 2016 ++++++++++++++++++++++ + +Client +====== + +Features +~~~~~~~~ +- Add recovery document format version for future migrations. +- Use DeferredLock instead of its locking cousin. +- Use DeferredSemaphore instead of its locking cousin. + +Bugfixes +~~~~~~~~ +- `#8180 `_: Initialize OpenSSL context just once. +- Remove document content conversion to unicode. Users of API are responsible + for only passing valid JSON to Soledad for storage. + +Misc +~~~~ +- Add ability to get information about sync phases for profiling purposes. +- Add script for setting up develop environment. +- Refactor bootstrap to remove shared db lock. +- Removed multiprocessing from encdecpool with some extra refactoring. +- Remove user_id argument from Soledad init. + +Common +====== + +Features +~~~~~~~~ +- Embed l2db, forking u1db. + +Misc +~~~~ +- Toxify tests. + 0.8.0 - 18 Apr, 2016 ++++++++++++++++++++ diff --git a/client/changes/next-changelog.rst b/client/changes/next-changelog.rst index 7ddb3a57..6c1c2a49 100644 --- a/client/changes/next-changelog.rst +++ b/client/changes/next-changelog.rst @@ -1,4 +1,4 @@ -0.8.1 - ... +0.8.2 - ... ++++++++++++++++++++ Please add lines to this file, they will be moved to the CHANGELOG.rst during @@ -10,26 +10,15 @@ I've added a new category `Misc` so we can track doc/style/packaging stuff. Features ~~~~~~~~ -- Add recovery document format version for future migrations. -- Use DeferredLock instead of its locking cousin. - `#1234 `_: Description of the new feature corresponding with issue #1234. -- New feature without related issue number. Bugfixes ~~~~~~~~ - `#1235 `_: Description for the fixed stuff corresponding with issue #1235. -- Remove document content conversion to unicode. Users of API are responsible - for only passing valid JSON to Soledad for storage. -- Bugfix without related issue number. Misc ~~~~ -- Add ability to get information about sync phases for profiling purposes. -- Add script for setting up develop environment. -- Refactor bootstrap to remove shared db lock. - `#1236 `_: Description of the new feature corresponding with issue #1236. -- Some change without issue number. -- Removed multiprocessing from encdecpool with some extra refactoring. Known Issues ~~~~~~~~~~~~ diff --git a/common/changes/next-changelog.rst b/common/changes/next-changelog.rst index c0974384..64162b7b 100644 --- a/common/changes/next-changelog.rst +++ b/common/changes/next-changelog.rst @@ -1,4 +1,4 @@ -0.8.0 - ... +0.8.2 - ... +++++++++++++++++++++++++++++++ Please add lines to this file, they will be moved to the CHANGELOG.rst during diff --git a/server/changes/next-changelog.rst b/server/changes/next-changelog.rst index bdc9f893..fc4cbc30 100644 --- a/server/changes/next-changelog.rst +++ b/server/changes/next-changelog.rst @@ -1,4 +1,4 @@ -0.8.1 - ... +0.8.2 - ... ++++++++++++++++++++ Please add lines to this file, they will be moved to the CHANGELOG.rst during -- cgit v1.2.3